From 147ea5e2eb047967f2d041a13af9fa8708982393 Mon Sep 17 00:00:00 2001 From: Kyne Date: Wed, 19 Apr 2017 14:08:52 +0800 Subject: [PATCH] #LMS-282 open source Juice --- .gitignore | 27 ++ README.md | 98 ++++ README_en.md | 112 +++++ doc/api_document.md | 177 +++++++ doc/juice_sdk_example.md | 106 ++++ doc/object.md | 56 +++ doc/params.md | 53 ++ juice-client-sdk/.gitignore | 26 + juice-client-sdk/pom.xml | 38 ++ .../juice/client/sdk/config/COMMON.java | 10 + .../sdk/exception/JuiceClientException.java | 16 + .../hujiang/juice/client/sdk/model/Kills.java | 70 +++ .../juice/client/sdk/model/Operations.java | 38 ++ .../juice/client/sdk/model/Querys.java | 78 +++ .../juice/client/sdk/model/Reconciles.java | 76 +++ .../juice/client/sdk/model/Submits.java | 167 +++++++ .../juice/client/sdk/package-info.java | 4 + .../juice/client/sdk/utils/JuiceClient.java | 49 ++ juice-common/.gitignore | 26 + juice-common/pom.xml | 99 ++++ .../hujiang/juice/common/config/COMMON.java | 15 + .../juice/common/config/Configure.java | 118 +++++ .../juice/common/error/CommonStatusCode.java | 79 +++ .../hujiang/juice/common/error/ErrorCode.java | 50 ++ .../common/exception/CacheException.java | 29 ++ .../common/exception/CommonException.java | 28 ++ .../exception/ConfigurationException.java | 28 ++ .../common/exception/DataBaseException.java | 29 ++ .../exception/HealthcheckException.java | 30 ++ .../exception/InternalServiceException.java | 36 ++ .../juice/common/exception/RestException.java | 29 ++ .../exception/UnauthorizedException.java | 29 ++ .../hujiang/juice/common/model/Command.java | 88 ++++ .../juice/common/model/Constraints.java | 44 ++ .../hujiang/juice/common/model/Container.java | 99 ++++ .../hujiang/juice/common/model/Docker.java | 145 ++++++ .../hujiang/juice/common/model/Expire.java | 27 ++ .../hujiang/juice/common/model/Resources.java | 123 +++++ .../com/hujiang/juice/common/model/Task.java | 85 ++++ .../juice/common/model/TaskManagement.java | 59 +++ .../hujiang/juice/common/package-info.java | 4 + .../hujiang/juice/common/utils/BeanUtil.java | 87 ++++ .../juice/common/utils/CommonUtils.java | 17 + .../hujiang/juice/common/utils/DESUtil.java | 103 ++++ .../hujiang/juice/common/utils/JsonUtils.java | 21 + .../juice/common/utils/cache/CacheUtils.java | 15 + .../common/utils/cache/RedisCacheUtils.java | 106 ++++ .../juice/common/utils/cache/RedisUtil.java | 454 ++++++++++++++++++ .../juice/common/utils/db/DaoUtils.java | 172 +++++++ .../juice/common/utils/db/JuiceDao.java | 165 +++++++ .../common/utils/generator/IdGenUtil.java | 74 +++ .../common/utils/generator/IdGenerator.java | 14 + .../juice/common/utils/rest/OkHttpUtils.java | 222 +++++++++ .../utils/rest/ParameterTypeReference.java | 65 +++ .../juice/common/utils/rest/Restty.java | 367 ++++++++++++++ .../com/hujiang/juice/common/vo/Result.java | 46 ++ .../juice/common/vo/ResultBuilder.java | 34 ++ .../hujiang/juice/common/vo/SubmitTask.java | 66 +++ .../com/hujiang/juice/common/vo/TaskKill.java | 20 + .../juice/common/vo/TaskReconcile.java | 37 ++ .../hujiang/juice/common/vo/TaskResult.java | 58 +++ juice-jooq/.gitignore | 26 + juice-jooq/pom.xml | 77 +++ juice-rest/pom.xml | 96 ++++ juice-rest/scripts/Dockerfile | 17 + juice-rest/scripts/entrypoint.sh | 7 + .../java/com/hujiang/juice/rest/Startup.java | 20 + .../hujiang/juice/rest/config/AppConfig.java | 76 +++ .../juice/rest/config/CacheConfig.java | 26 + .../juice/rest/config/CachesBizConfig.java | 33 ++ .../com/hujiang/juice/rest/package-info.java | 4 + .../juice/rest/service/RestService.java | 166 +++++++ .../juice/rest/service/SchedulerService.java | 22 + .../juice/rest/utils/SubscriberUtils.java | 131 +++++ .../hujiang/juice/rest/utils/TaskUtils.java | 140 ++++++ .../web/controller/BaseAppController.java | 217 +++++++++ .../rest/web/controller/BaseController.java | 147 ++++++ .../rest/web/controller/TaskController.java | 98 ++++ .../config/application-dev.properties | 12 + .../resources/config/application.properties | 22 + .../src/main/resources/config/logback-dev.xml | 58 +++ .../src/main/resources/config/logback.xml | 57 +++ juice-service/.gitignore | 26 + juice-service/pom.xml | 95 ++++ juice-service/scripts/Dockerfile | 17 + juice-service/scripts/entrypoint.sh | 7 + .../com/hujiang/juice/service/Startup.java | 45 ++ .../hujiang/juice/service/config/JUICE.java | 162 +++++++ .../juice/service/driver/SchedulerDriver.java | 286 +++++++++++ .../service/exception/DriverException.java | 18 + .../service/exception/UnrecoverException.java | 23 + .../service/factory/DataSourceFactory.java | 128 +++++ .../juice/service/factory/JuiceFactory.java | 43 ++ .../juice/service/factory/RedisFactory.java | 72 +++ .../com/hujiang/juice/service/model/Host.java | 17 + .../juice/service/model/SchedulerCalls.java | 159 ++++++ .../juice/service/model/SendErrors.java | 22 + .../hujiang/juice/service/package-info.java | 1 + .../service/service/AuxiliaryService.java | 348 ++++++++++++++ .../service/service/SchedulerService.java | 344 +++++++++++++ .../juice/service/support/Support.java | 22 + .../juice/service/utils/LogInitUtil.java | 40 ++ .../juice/service/utils/ResourcesUtils.java | 36 ++ .../juice/service/utils/SendUtils.java | 59 +++ .../service/utils/protocol/Protobuf.java | 34 ++ .../service/utils/protocol/Protocol.java | 14 + .../service/utils/zookeeper/CuratorUtils.java | 187 ++++++++ .../service/utils/zookeeper/JsonInfo.java | 39 ++ .../utils/zookeeper/LeaderSelectorClient.java | 50 ++ .../main/resources/application-dev.properties | 18 + .../src/main/resources/logback-dev.xml | 56 +++ pom.xml | 252 ++++++++++ script/juice_2017-V1.0-OPEN.sql | 81 ++++ 113 files changed, 8796 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 README_en.md create mode 100644 doc/api_document.md create mode 100644 doc/juice_sdk_example.md create mode 100644 doc/object.md create mode 100644 doc/params.md create mode 100644 juice-client-sdk/.gitignore create mode 100644 juice-client-sdk/pom.xml create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/config/COMMON.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/exception/JuiceClientException.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Kills.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Operations.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Querys.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Reconciles.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Submits.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/package-info.java create mode 100644 juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/utils/JuiceClient.java create mode 100644 juice-common/.gitignore create mode 100644 juice-common/pom.xml create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/config/COMMON.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/config/Configure.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/error/CommonStatusCode.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/error/ErrorCode.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/CacheException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/CommonException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/ConfigurationException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/DataBaseException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/HealthcheckException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/InternalServiceException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/RestException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/exception/UnauthorizedException.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Command.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Constraints.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Container.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Docker.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Expire.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Resources.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/Task.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/model/TaskManagement.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/package-info.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/BeanUtil.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/CommonUtils.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/DESUtil.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/JsonUtils.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/cache/CacheUtils.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisCacheUtils.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisUtil.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/db/DaoUtils.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/db/JuiceDao.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenUtil.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenerator.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/rest/OkHttpUtils.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/rest/ParameterTypeReference.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/utils/rest/Restty.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/vo/Result.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/vo/ResultBuilder.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/vo/SubmitTask.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/vo/TaskKill.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/vo/TaskReconcile.java create mode 100644 juice-common/src/main/java/com/hujiang/juice/common/vo/TaskResult.java create mode 100644 juice-jooq/.gitignore create mode 100644 juice-jooq/pom.xml create mode 100644 juice-rest/pom.xml create mode 100644 juice-rest/scripts/Dockerfile create mode 100644 juice-rest/scripts/entrypoint.sh create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/Startup.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/config/AppConfig.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/config/CacheConfig.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/config/CachesBizConfig.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/package-info.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/service/RestService.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/service/SchedulerService.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/utils/SubscriberUtils.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/utils/TaskUtils.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseAppController.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseController.java create mode 100644 juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/TaskController.java create mode 100644 juice-rest/src/main/resources/config/application-dev.properties create mode 100644 juice-rest/src/main/resources/config/application.properties create mode 100644 juice-rest/src/main/resources/config/logback-dev.xml create mode 100644 juice-rest/src/main/resources/config/logback.xml create mode 100644 juice-service/.gitignore create mode 100644 juice-service/pom.xml create mode 100644 juice-service/scripts/Dockerfile create mode 100644 juice-service/scripts/entrypoint.sh create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/Startup.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/config/JUICE.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/driver/SchedulerDriver.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/exception/DriverException.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/exception/UnrecoverException.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/factory/DataSourceFactory.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/factory/JuiceFactory.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/factory/RedisFactory.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/model/Host.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/model/SchedulerCalls.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/model/SendErrors.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/package-info.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/service/AuxiliaryService.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/service/SchedulerService.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/support/Support.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/LogInitUtil.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/ResourcesUtils.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/SendUtils.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protobuf.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protocol.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/CuratorUtils.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/JsonInfo.java create mode 100644 juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/LeaderSelectorClient.java create mode 100644 juice-service/src/main/resources/application-dev.properties create mode 100644 juice-service/src/main/resources/logback-dev.xml create mode 100644 pom.xml create mode 100644 script/juice_2017-V1.0-OPEN.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96c6f38 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Created by .ignore support plugin (hsz.mobi) +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### Example user template template +### Example user template + +# IntelliJ project files +.idea/ +.settings/ +target/ +.project +.classpath +*.iml +out +gen +log/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcba37e --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +For [ENGLISH](README_en.md) +## Why Juice +juice是沪江(hujiang)学习系统项目组(LMS)所开发的一套基于Mesos Framework的分布式任务调度云系统，基于此系统，可以实现任何作业型任务的调度工作。 + +耗时计算型任务处理在沪江一直是个持续的需求，例如多媒体转码，Map Reduce，密集计算等等任务。在Juice出现以前，各个项目组各显神通，开发着不同的任务处理系统，不但耗费人力，还特别耗费服务器。 +Juice利用Mesos集群空闲的计算能力，负责统一的接口返回和任务查询功能，其优点如下： +* 基于Mesos集群，最大程度利用空闲资源完成耗时计算 +* 异步调度，并提供结果查询和回调两种方法获取执行结果 +* 分布式管理，所有耗时任务都在资源池中无状态调度。 +* 任务查询，目前只开放通过接口查询任务的状态和结果。 + + +##### Latest Release Version:1.0-OPEN + +## Getting Started +##### Requirements +* [Mesos](http://mesos.apache.org/gettingstarted/) +* [ZooKeeper](https://zookeeper.apache.org/doc/r3.4.6/zookeeperStarted.html) +* Java8 +* [MySql](https://dev.mysql.com/doc/mysql-getting-started/en/) +* [Redis](https://redis.io/) +* [Marathon](https://mesosphere.github.io/marathon/)(optional) +* [Docker](https://www.docker.com/) + +## Install +* 安装zookeeper集群 +* 安装Mesos集群，并加入Zookeeper集群中。 + +~~~~ + 配置zookeeper地址到项目配置文件中: + 1.juice-service/src/main/resources/application-dev.properties + mesos.framework.zk=your_zk1_ip:port,your_zk2_ip:port,your_zk3_ip:port,your_zk4_ip:port,your_zk5_ip:port + PS juice-service通过zookeeper获取mesos master地址，所以必须将mesos集群配置到zookeeper中进行HA切换。 +~~~~ + +* 安装mysql5.6以上版本 + +~~~~ + 创建mysql数据juice,schema为juice。 + 执行/script/juice_2017-V1.0-OPEN.sql创建2张表juice_framework、juice_task。 + 修改以下内容： + 1.juice-jooq/pom.xml： + + com.mysql.jdbc.Driver + jdbc:mysql://your_ip:port/juice + user + password + + 2.juice-rest/src/main/resources/config/application-dev.properties + spring.datasource.url=jdbc:mysql://your_ip:port/juice?useUnicode=true&characterEncoding=utf-8&autoReconnect=true&useSSL=false + spring.datasource.username=user + spring.datasource.password=password + 3.juice-service/src/main/resources/application-dev.properties + db.url=jdbc:mysql://your_ip:port/juice?useUnicode=true&characterEncoding=utf-8&autoReconnect=true&useSSL=false + db.user=user + db.password=password + 将以上内容修改为数据库安装时的配置信息。 +~~~~ + +* 安装Redis3.0以上版本 + +~~~~ + 修改以下内容： + 1.juice-rest/src/main/resources/config/application-dev.properties + spring.redis.host=redis_ip + spring.redis.port=redis_port + spring.redis.password= + 2.juice-service/src/main/resources/application-dev.properties + redis.host=redis_ip + redis.port=redis_port + redis.password= + 将以上内容修改为Redis安装时的配置信息。 +~~~~ + +~~~~ +在项目根目录执行mvn clean install编译项目，编译后产生juice-service和juice-rest2个jar包。 +项目启动: +* java -Dsystem.environment=dev -jar juice-service.jar +* java -Dspring.profiles.active=dev -jar juice-rest.jar +~~~~ + +## How To Use Juice### +* [API Document](doc/api_document.md) +* [Juice_SDK示例](doc/juice_sdk_example.md) +* [参数说明文档](doc/params.md) + +创建任务步骤: +* 开发能够使用Docker启动的任务（主业务逻辑）并PUSH到Docker镜像仓库。 +* 调用Juice API发起任务 +* 等待回调结果，或主动fetch结果 + +~~~~ +Juice最适合运行Docker任务,通常做法将Jar包打入一个Docker镜像，提交此镜像到dockerhub上，提交一个Container模式的任务到到Juice,其中参数dockerImage为Docker镜像名。 +Juice会自动的寻找一个合适的Agent来执行此任务，调用方不必关心在哪个Host上执行，而只需要关心任务是否执行成功即可。 +~~~~ + + + diff --git a/README_en.md b/README_en.md new file mode 100755 index 0000000..871a038 --- /dev/null +++ b/README_en.md @@ -0,0 +1,112 @@ +## Why Juice + +Juice is a set of distributed task scheduling cloud system based on Mesos Framework developed by Hujiang Learning Management System group (LMS). Based on this system you can realize the scheduling of any job task. + +Time-consuming computing tasks in Hujiang is a continuous demand. Such as multimedia transcoding, Map Reduce, intensive computing and so on. Each working team were do their own before Juice, do the development of different task processing systems, not only consuming the manpower, but also consuming the server. + +Juice uses Mesos clustered idle computing capabilities, responsible for a unified interface return and task query function, the advantages are as follows: + +* Based on the Mesos cluster, the maximum use of idle resources to complete time-consuming calculation +* Asynchronous scheduling, and provide results query and callback to obtain the results of the implementation +* Distributed management, all time-consuming tasks are stateless in resource pools. +* Task query, currently opened the interface for query task status and results. + +##### Latest Release Version:1.0-OPEN + +## Getting Started + +##### Requirements + +* [Mesos](http://mesos.apache.org/gettingstarted/) +* [ZooKeeper](https://zookeeper.apache.org/doc/r3.4.6/zookeeperStarted.html) +* Java8 +* [MySql](https://dev.mysql.com/doc/mysql-getting-started/en/) +* [Redis](https://redis.io/) +* [Marathon](https://mesosphere.github.io/marathon/)(optional) +* [Docker](https://www.docker.com/) + +## Install + +* Install the Zookeeper cluster +* Install the Mesos cluster and join the Zookeeper cluster. + +~~~~ + Add Zookeeper's address to the configuration file of project: + + juice-service/src/main/resources/application-dev.properties + mesos.framework.zk=your_zk1_ip:port,your_zk2_ip:port,your_zk3_ip:port,your_zk4_ip:port,your_zk5_ip:port + + NOTE: juice-service will get Mesos Master's address through Zookeeper, must add Mesos cluster to Zookeeper cluster to do the HA switch. +~~~~ + +* Install mysql5.6 or later + +~~~~ + Create mysql database juice and schema as juice. + Run /script/juice_2017-V1.0-OPEN.sql to create tables juice_framework and juice_task. + + Modify the configuration: + + 1.juice-jooq/pom.xml： + + com.mysql.jdbc.Driver + jdbc:mysql://your_ip:port/juice + user + password + + + 2.juice-rest/src/main/resources/config/application-dev.properties + spring.datasource.url=jdbc:mysql://your_ip:port/juice?useUnicode=true&characterEncoding=utf-8&autoReconnect=true&useSSL=false + spring.datasource.username=user + spring.datasource.password=password + + 3.juice-service/src/main/resources/application-dev.properties + db.url=jdbc:mysql://your_ip:port/juice?useUnicode=true&characterEncoding=utf-8&autoReconnect=true&useSSL=false + db.user=user + db.password=password + +~~~~ + +* Install Redis3.0 or later + +~~~~ + Modify the configuration: + + 1.juice-rest/src/main/resources/config/application-dev.properties + spring.redis.host=redis_ip + spring.redis.port=redis_port + spring.redis.password= + + 2.juice-service/src/main/resources/application-dev.properties + redis.host=redis_ip + redis.port=redis_port + redis.password= + +~~~~ + +~~~~ +Run mvn clean install at root path of project and compile. juice-service and juice-rest will be generated after compile. + +Start Project: + +* java -Dsystem.environment=dev -jar juice-service.jar +* java -Dspring.profiles.active=dev -jar juice-rest.jar +~~~~ + +## How To Use + +* [API Document](doc/api_document.md) +* [Juice_SDK Sample](doc/juice_sdk_example.md) +* [Parameter Description](doc/params.md) + +Step of creating task: + +* Develop the ability to use Docker to start the task (main business logic) and push to Docker repository. +* Call Juice API with the task description(BODY) +* Wait for the results callback or fetch results manually. + +~~~~ +Juice is best suited for running Docker tasks, and it is common practice to put a Jar package into a Docker image. Submit the mirror to Dockerhub and submit a container mode task to Juice where the dockerImage parameter is the Docker mirror name. + +Juice will automatically find a suitable Agent to perform this task, the caller does not have to care about which host to perform on, but only need to care about whether the task can be successful. +~~~~ diff --git a/doc/api_document.md b/doc/api_document.md new file mode 100644 index 0000000..a7369c9 --- /dev/null +++ b/doc/api_document.md @@ -0,0 +1,177 @@ +## Request Header + +| name | type | is Required | Description | +| ------------- |:-------------:| ------------:| --------------:| +| X-Tenant-Id | String | Y | 请求方系统标示号 | + +#### 1.提交一个任务 +URL:/v1/tasks +Method : POST + +## RequestBody + +| name | type | is Required | Description | +| ------------- |:-------------:| -----------------------:| --------------:| +| taskName | String | Y | 任务名称 | +| callbackUrl | String | N | 任务回调URL | +| resources | Resources | N | 任务所需硬件资源 | +| container | Container | N(运行Container任务时非空)| Container任务 | +| commands | String | N(运行Commands任务时非空) | Commands任务 | +| env | Environment | N | 环境变量 | +| args | List | N(Container任务时可传字段)| 用户参数 | +| constraints | Constraints | N | 约束 | + +ps: type中的对象定义参见[对象定义说明](object.md) +
+example run container:
+{  
+   "callbackUrl":"http://www.XXXXXXXX.com/v5/tasks/callback",	    
+   "taskName":"demo-container",									        
+   "env":{"name":"environment","value":"dev"},   				
+    "args":["this is a test"],                     			
+    "container":{       		
+      	"docker":{
+        	"image":"dockerhub.XXXX.com/demo-slice"        	
+      	},
+    	"type":"DOCKER"
+    }
+}
+
+
+example run commands:
+{  
+    "callbackUrl":"http://www.XXXXXXXX.com/v5/tasks/callback",
+	"taskName":"demo-commands",									
+	"env":{"name":"environment","value":"dev"},   				
+  	"commands":"/home/app/entrypoint.sh"						
+}
+
+ +## ResponseBody + +| name | type | is Required | Description | +| ------------- |:-------------:| -----------------------:| --------------:| +| taskId | Long | Y | Juice任务ID | + +
+response ok, httpcode 200:
+{  
+	"status": 0,
+	"message":"OK",
+	"data" :{
+		"taskId": 806769905152840693
+	}						
+}
+
+response failed, httpcode 400/500:
+{
+	"status": -2127552257,
+	"message":"object not null error"
+}
+
+ +#### 2.(批量)查询任务状态 +URL:/v1/tasks?taskIds={ids} +Method : GET + +## RequestParams + +| name | type | is Required | Description | +| ------------- |:-------------:| -----------------------:| -------------------------:| +| taskIds | String | Y | Juice任务IDs(Id间以逗号分割)| + +## ResponseBody + +| name | type | is Required | Description | +| ------------- |:-------------:| -----------------------:| -------------------------:| +| taskResults | List | Y | 任务信息列表 | + +
+response ok, httpcode 200:
+{  
+"status": 0,
+	"message":"OK",
+	"data" :{
+		[{
+			"taskId": 806769905152840693,
+			"taskName": "test-docker",
+			"tenantId": "10002",
+			"dockerImage": "dockerhub.XXXX.com/juice-test",
+			"taskStatus": 2,									//	参考taskStatus说明
+			"message": "Container exited with status 0",
+			"callbackAt": "Dec 8, 2016 4:03:08 PM",
+			"callbackUrl": "http://localhost:9999/v1/callback",
+			"submitAt": "Dec 8, 2016 3:58:29 PM"
+		},
+		{
+			"taskId": 806769905152840695,
+			"taskName": "test-commands",
+			"tenantId": "10002",
+			"commands": "/home/app/entrypoint.sh",
+			"taskStatus": 3,
+			"message": "Container exited with status 0",
+			"callbackAt": "Dec 8, 2016 4:04:08 PM",
+			"callbackUrl": "http://localhost:9999/v1/callback",
+			"submitAt": "Dec 8, 2016 3:54:29 PM"
+		}]
+	}						
+}
+
+response failed, httpcode 400/500:
+{
+	"status": -2127552257,
+	"message":"object not null error"
+}
+
+ +#### 3.终止一个任务 + +URL:/v1/tasks/kill?taskId={id} +Method : POST + +## RequestParams + +| name | type | is Required | Description | +| ------------- |:-------------:| -----------------------:| -------------------------:| +| taskId | Long | Y | Juice任务ID | + +## ResponseBody + +| name | type | is Required | Description | +| ------------- |:-------------:| -----------------------:| -------------------------:| +| submitKill | Boolean | Y | 是否提交删除 | +| resultMessage | String | Y | 描述信息 | +| currentStatus | byte | Y | 当前状态 | +| | | | NOT_START(-1), | +| | | | STAGING(0), | +| | | | RUNNING(1), | +| | | | FINISHED(2), | +| | | | FAILED(3), | +| | | | LOST(4), | +| | | | ERROR(5), | +| | | | KILLED(6), | +| | | | UNREACHABLE(7), | +| | | | DROPPED(8), | +| | | | GONE(9), | +| | | | GONE_BY_OPERATOR(10), | +| | | | UNKNOWN(11), | +| | | | EXPIRED(12); | + +
+response ok, httpcode 200:
+{  
+	"status": 0,
+	"message":"OK",
+	"data" :{
+		"submitKill":true,
+		"resultMessage":"task is killed",
+		"currentStatus":6
+	}					
+}
+
+response failed, httpcode 400/500:
+{
+	"status": -2127552257,
+	"message":"object not null error"
+}
+
\ No newline at end of file diff --git a/doc/juice_sdk_example.md b/doc/juice_sdk_example.md new file mode 100644 index 0000000..7645e04 --- /dev/null +++ b/doc/juice_sdk_example.md @@ -0,0 +1,106 @@ +##### Juice client sdk采用java1.8编写，Maven依赖： + + + com.hujiang + juice-client-sdk + 1.0-OPEN + + + +
+提交一个Docker任务
+@Test
+public void submitsDocker() {
+    string tenantId = "ocs_test";
+    Submits submitsDocker = Submits.create()
+            .setDockerImage("dockerhub.XXXX.com/demo-slice")
+            .setTaskName("demo-slice")
+            .addArgs("/10002/res/L2.mp4")
+            .addEnv("environment", "dev")
+            .addResources(2.0, 2048.0);
+ 
+    Long taskId = JuiceClient.create(juiceRequestUrl, tenantId)
+            .setOperations(submitsDocker)
+            .handle();
+ 
+    if(null != taskId) {
+        System.out.println("submitsDocker, taskId --> " + taskId);
+    }
+}
+
+ +
+提交一个Commands任务
+@Test
+public void submitsCommands() {
+    string tenantId = "ocs_test";
+    Submits submitsCommands = Submits.create()
+            .setCommands("/home/app/commands-test/entrypoint.sh")
+            .setTaskName("test-commands")
+            .addConstraints(Constraints.FIELD.HOSTNAME, "192.168.0.1")           //   Constraints目前支持2种约束模式，hostname/rack_id
+            .addConstraints(Constraints.FIELD.HOSTNAME, "192.168.0.2")           //   当选用HOSTNAME模式时,juice会从符合的N个HOSTNAME中选取一个执行任务       
+            .addConstraints(Constraints.FIELD.HOSTNAME, "192.168.0.3")           //   当选用RACK_ID模式时,juice会从符合该rack_id的Agent中选取一个执行任务，关于rack_id需要在mesos上设置attribute.
+            .addResources(0.5, 256.0);
+ 
+    Long taskId = JuiceClient.create(juiceRequestUrl, tenantId)
+            .setOperations(submitsCommands)
+            .handle();
+ 
+    if(null != taskId) {
+        System.out.println("submitsCommands, taskId --> " + taskId);
+    }
+}
+
+ + +
+查询任务状态
+@Test
+public void querys() {
+    string tenantId = "ocs_test";
+    Querys querys = Querys.create()
+                .addTask(832419514512333559L);
+    List tasks = JuiceClient.create(juiceRequestUrl, tenantId)
+                .setOperations(querys)
+                .handle();
+     
+    if (null != tasks && tasks.size() > 0) {
+        System.out.println("query tasks : "
+            + tasks.stream().map(Task::toString).collect(Collectors.joining(",", "{\n\t\t", "\n}")));
+    }
+}
+
+ +
+终止一个正在执行的任务
+@Test
+public void kills() {
+    string tenantId = "ocs_test";
+    Kills kills = Kills.create()
+            .setTaskId(832419514512333759L);
+    TaskKill killed = JuiceClient.create(juiceRequestUrl, ocs_test)
+                .setOperations(kills)
+                .handle();
+    if(null != killed) {
+        System.out.println("submitsDocker, is killed ? --> " + killed.toString());
+    }
+}
+
+ +
+同步一个任务的状态
+@Test
+public void reconciles() {
+    string tenantId = "ocs_test";
+    Reconciles reconciles = Reconciles.create()
+                    .addTask(832419514512333759L);
+ 
+    TaskReconcile taskReconcile = JuiceClient.create(juiceRequestUrl, tenantId)
+                    .setOperations(reconciles)
+                    .handle();
+ 
+    if(null != taskReconcile) {
+        System.out.println("reconciles, taskReconcile --> " + taskReconcile);
+    }
+}
+
\ No newline at end of file diff --git a/doc/object.md b/doc/object.md new file mode 100644 index 0000000..6cf22d1 --- /dev/null +++ b/doc/object.md @@ -0,0 +1,56 @@ +
+class Resources {
+    String role;
+    int cpu;
+    int mem;
+}
+
+ +
+class Environment {
+    String name;
+    String value;
+}
+
+ +
+class Constraints {
+    String field;           //  "RACK_ID"/"HOSTNAME"
+    Set values;
+}
+
+ + +
+//  目前Container只支持运行Docker模式
+class Container {
+    Docker docker;
+    String type;            //  "DOCKER"/"MESOS"
+    List volumes;
+ 
+    class Volume {
+        String containerPath;
+        String hostPath;
+        String dvo;         //  "RO"/"RW"
+    }
+  
+    class Docker {
+        String image;
+        boolean forcePullImage;
+        boolean privileged;
+        String net;                     //  "BRIDGE"/"HOST"/"NONE"/"USER"
+        List parameters;
+        List portMappings;
+ 
+        class PortMapping {
+            int containerPort;
+            int hostPort;
+            String protocol;
+        }
+        class Parameter {
+            String key;
+            String value;
+        }
+    }
+}
+
\ No newline at end of file diff --git a/doc/params.md b/doc/params.md new file mode 100644 index 0000000..75ec0dd --- /dev/null +++ b/doc/params.md @@ -0,0 +1,53 @@ +### Params +Juice所有的参数均已设置默认值，同时也提供用户自定义输入参数参数值，只需在配置文件中加个具体参数值即可 + +#### Juice Rest +设置参数到(dev)： +juice-rest/src/main/resources/config/application-dev.properties + +~~~~ +#任务处理队列,默认为juice.task.queue.dev,如设置,必须和juice-service中设置相同 +juice.task.queue＝ +#任务结果队列,默认为juice.task.result.queue.dev,必须和juice-service中设置相同 +juice.task.result.queue= +#任务管理队列,默认为juice.management.queue,必须和juice-service中设置相同 +juice.management.queue= +#任务在队列中的过期时间,默认86400,如设置,必须和juice-rest中设置相同 +juice.task.expired.of.seconds= +~~~~ + +#### Juice Service +设置参数到(dev)： +juice-service/src/main/resources/application-dev.properties + +~~~~ +#Zookeeper集群地址 +mesos.framework.zk＝ +#是否使用zookeeper进行HA,默认true +zookeeper.distribute.lock.ha= +#framework的名字,默认以juice-service-dev开头 +mesos.scheduler.name= +#单个Agent资源分配阀值,默认0.8 +resources.use.threshold= +#指定Mesos的Attribute,使juice-service只使用mesos集群中的某些资源 +#比如ocs|ams,qa,mid +mesos.framework.attr= +#send-pool-size,默认20 +send.pool.size= +#auxiliary-pool-size,默认20 +auxiliary.pool.size= +＃max reserved，默认1024，任务重试次数 +max.reserved= +#framework的标记,默认mesos.framework.tag.dev +mesos.framework.tag= +#任务处理队列,默认为juice.task.queue.dev,如设置,必须和juice-rest中设置相同 +juice.task.queue＝ +#任务结果队列,默认为juice.task.result.queue.dev,如设置,必须和juice-rest中设置相同 +juice.task.result.queue= +#任务管理队列,默认为juice.management.queue,如设置,必须和juice-rest中设置相同 +juice.management.queue= +#重试队列,默认为juice.task.retry.queue.dev +juice.task.retry.queue= +#任务在队列中的过期时间,默认86400,如设置,必须和juice-rest中设置相同 +juice.task.expired.of.seconds= +~~~~ diff --git a/juice-client-sdk/.gitignore b/juice-client-sdk/.gitignore new file mode 100644 index 0000000..ce2fd51 --- /dev/null +++ b/juice-client-sdk/.gitignore @@ -0,0 +1,26 @@ +# Created by .ignore support plugin (hsz.mobi) +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### Example user template template +### Example user template + +# IntelliJ project files +.idea/ +.settings/ +target/ +.project +.classpath +*.iml +out +gen diff --git a/juice-client-sdk/pom.xml b/juice-client-sdk/pom.xml new file mode 100644 index 0000000..7cb3870 --- /dev/null +++ b/juice-client-sdk/pom.xml @@ -0,0 +1,38 @@ + + + + juice + com.hujiang + 1.0-OPEN + + 4.0.0 + + juice-client-sdk + + + + com.hujiang + juice-common + + + com.hujiang + juice-jooq + + + + juice-common + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + + + + + \ No newline at end of file diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/config/COMMON.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/config/COMMON.java new file mode 100644 index 0000000..e61e1ec --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/config/COMMON.java @@ -0,0 +1,10 @@ +package com.hujiang.juice.client.sdk.config; + +/** + * Created by xujia on 17/2/13. + */ +public class COMMON { + public static final String URL_KILL = "/kill"; + public static final String URL_RECONCILE = "/reconcile"; + public static final String TENANT_ID_HEAD = "X-Tenant-Id"; +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/exception/JuiceClientException.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/exception/JuiceClientException.java new file mode 100644 index 0000000..ed59dcf --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/exception/JuiceClientException.java @@ -0,0 +1,16 @@ +package com.hujiang.juice.client.sdk.exception; + + +import com.hujiang.juice.common.exception.CommonException; + +/** + * Created by xujia on 17/2/6. + */ + +public class JuiceClientException extends CommonException { + + public JuiceClientException(int code, String message){ + super(code, message); + } + +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Kills.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Kills.java new file mode 100644 index 0000000..6258d78 --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Kills.java @@ -0,0 +1,70 @@ +package com.hujiang.juice.client.sdk.model; + + +import com.google.common.collect.Maps; +import com.hujiang.juice.client.sdk.exception.JuiceClientException; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.utils.rest.ParameterTypeReference; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.common.vo.Result; +import com.hujiang.juice.common.vo.TaskKill; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.Map; + +import static com.hujiang.juice.client.sdk.config.COMMON.*; + +/** + * Created by xujia on 17/2/13. + */ + +@Data +@Slf4j +@EqualsAndHashCode(callSuper = false) +public class Kills extends Operations { + private long taskId; + + private Kills(long taskId) { + this.taskId = taskId; + } + + private Kills() { + + } + + public static Kills create(long taskId) { + return new Kills(taskId); + } + + public static Kills create() { + return new Kills(); + } + + public Kills setTaskId(long taskId) { + this.taskId = taskId; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public TaskKill handle(String requestUrl, String tenantId) { + Result result = null; + try { + Map map = Maps.newHashMap(); + map.put("taskId", String.valueOf(taskId)); + String url = requestUrl + URL_KILL; + result = Restty.create(url, map) + .addMediaType(Restty.jsonBody()) + .addHeader(TENANT_ID_HEAD, tenantId) + .post(new ParameterTypeReference>() { + }); + } catch (IOException e) { + throw new JuiceClientException(ErrorCode.HTTP_REQUEST_ERROR.getCode(), e.getMessage()); + } + + return result != null ? result.getData() : null; + } +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Operations.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Operations.java new file mode 100644 index 0000000..6e1cd10 --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Operations.java @@ -0,0 +1,38 @@ +package com.hujiang.juice.client.sdk.model; + +import com.google.common.collect.Maps; +import com.hujiang.juice.client.sdk.exception.JuiceClientException; +import com.hujiang.juice.common.error.ErrorCode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Created by xujia on 17/2/15. + */ +@Slf4j +public abstract class Operations { + + public abstract T handle(String requestUrl, String tenantId); + + private static String idsToString(List ids) { + return ids.stream() + .map(String::valueOf) + .collect(Collectors.joining(",")); + } + + Map getTaskIdsStr(List taskIdList) { + String ids = idsToString(taskIdList); + if(StringUtils.isBlank(ids)) { + String message = "task-ids is null!"; + log.warn(message); + throw new JuiceClientException(ErrorCode.OBJECT_NOT_NULL_ERROR.getCode(), message); + } + Map map = Maps.newHashMap(); + map.put("taskIds", ids); + return map; + } +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Querys.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Querys.java new file mode 100644 index 0000000..0b8a038 --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Querys.java @@ -0,0 +1,78 @@ +package com.hujiang.juice.client.sdk.model; + +import com.google.common.collect.Lists; + +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.client.sdk.exception.JuiceClientException; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.utils.rest.ParameterTypeReference; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.common.vo.Result; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static com.hujiang.juice.client.sdk.config.COMMON.*; + +/** + * Created by xujia on 17/2/13. + */ + +@Data +@Slf4j +@EqualsAndHashCode(callSuper = false) +public class Querys extends Operations{ + private List taskIdList; + + private Querys() { + taskIdList = Lists.newArrayList(); + } + + private Querys(List taskIdList) { + this.taskIdList = taskIdList; + } + + public static Querys create() { + return new Querys(); + } + + public static Querys create(List taskIdList) { + return new Querys(taskIdList); + } + + public Querys addTask(long id) { + taskIdList.add(id); + return this; + } + + public Querys addAllTask(List ids) { + taskIdList.addAll(ids); + return this; + } + + @Override + @SuppressWarnings("unchecked") + public List handle(String requestUrl, String tenantId) { + + Map map = getTaskIdsStr(taskIdList); + + Result> result = null; + try { + result = Restty.create(requestUrl, map) + .addMediaType(Restty.jsonBody()) + .addHeader(TENANT_ID_HEAD, tenantId) + .get(new ParameterTypeReference>>() { + }); + + } catch (IOException e) { + throw new JuiceClientException(ErrorCode.HTTP_REQUEST_ERROR.getCode(), e.getMessage()); + } + + return result != null ? result.getData() : null; + } + +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Reconciles.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Reconciles.java new file mode 100644 index 0000000..9620fe9 --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Reconciles.java @@ -0,0 +1,76 @@ +package com.hujiang.juice.client.sdk.model; + +import com.google.common.collect.Lists; +import com.hujiang.juice.client.sdk.exception.JuiceClientException; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.utils.rest.ParameterTypeReference; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.common.vo.Result; +import com.hujiang.juice.common.vo.TaskReconcile; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static com.hujiang.juice.client.sdk.config.COMMON.*; + +/** + * Created by xujia on 17/2/13. + */ + +@Data +@Slf4j +@EqualsAndHashCode(callSuper = false) +public class Reconciles extends Operations{ + private List taskIdList; + + private Reconciles() { + taskIdList = Lists.newArrayList(); + } + + private Reconciles(List taskIdList) { + this.taskIdList = taskIdList; + } + + public static Reconciles create() { + return new Reconciles(); + } + + public static Reconciles create(List taskIdList) { + return new Reconciles(taskIdList); + } + + public Reconciles addTask(long id) { + taskIdList.add(id); + return this; + } + + public Reconciles addAllTask(List ids) { + taskIdList.addAll(ids); + return this; + } + + @Override + @SuppressWarnings("unchecked") + public TaskReconcile handle(String requestUrl, String tenantId) { + + Map map = getTaskIdsStr(taskIdList); + + Result result = null; + try { + String url = requestUrl + URL_RECONCILE; + result = Restty.create(url, map) + .addMediaType(Restty.jsonBody()) + .addHeader(TENANT_ID_HEAD, tenantId) + .post(new ParameterTypeReference>() { + }); + } catch (IOException e) { + throw new JuiceClientException(ErrorCode.HTTP_REQUEST_ERROR.getCode(), e.getMessage()); + } + + return result != null ? result.getData() : null; + } +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Submits.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Submits.java new file mode 100644 index 0000000..8eaeddf --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/model/Submits.java @@ -0,0 +1,167 @@ +package com.hujiang.juice.client.sdk.model; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.hujiang.juice.client.sdk.exception.JuiceClientException; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.model.*; + +import com.hujiang.juice.common.utils.rest.ParameterTypeReference; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.common.vo.Result; +import com.hujiang.juice.common.vo.SubmitTask; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import static com.hujiang.juice.client.sdk.config.COMMON.*; + +/** + * Created by xujia on 17/2/13. + */ + +@Slf4j +@Data +@EqualsAndHashCode(callSuper = false) +public class Submits extends Operations{ + private SubmitTask submitTask; + + private Submits() { + submitTask = new SubmitTask(); + } + private Submits(SubmitTask requestTask) { + this.submitTask = requestTask; + } + + public static Submits create() { + return new Submits(); + } + + public static Submits create(SubmitTask requestTask) { + return new Submits(requestTask); + } + + public Submits setDocker(Docker docker) { + if (null != submitTask.getRunMode()) { + log.info("already set run mode : " + submitTask.getRunMode() + ", can't set it again!"); + } + submitTask.setRunMode(SubmitTask.RunModel.CONTAINER); + submitTask.setContainer(new Container(docker)); + return this; + } + + public Submits setDockerImage(String image) { + if (null != submitTask.getRunMode()) { + log.info("already set run mode : " + submitTask.getRunMode() + ", can't set it again!"); + } + submitTask.setRunMode(SubmitTask.RunModel.CONTAINER); + submitTask.setContainer(new Container(new Docker(image))); + return this; + } + + public Submits setDockerImage(String image, boolean forcePullImage) { + if (null != submitTask.getRunMode()) { + log.info("already set run mode : " + submitTask.getRunMode() + ", can't set it again!"); + } + submitTask.setRunMode(SubmitTask.RunModel.CONTAINER); + submitTask.setContainer(new Container(new Docker(image, forcePullImage))); + return this; + } + + public Submits setCommands(String commands) { + if (null != submitTask.getRunMode()) { + log.info("already set run mode : " + submitTask.getRunMode() + ", can't set it again!"); + } + submitTask.setRunMode(SubmitTask.RunModel.COMMAND); + submitTask.setCommands(commands); + return this; + } + + public Submits setTaskName(String taskName) { + submitTask.setTaskName(taskName); + return this; + } + + public Submits setCallBackUrl(String callBackUrl) { + submitTask.setCallbackUrl(callBackUrl); + return this; + } + + public Submits addEnv(String k, String v) { + submitTask.setEnv(new Command.Environment(k, v)); + return this; + } + + public Submits setConstraints(Constraints.FIELD key, Set values) { + submitTask.setConstraints(new Constraints(key.getField(), values)); + return this; + } + + public Submits addConstraints(Constraints.FIELD key, String value) { + if(null == submitTask.getConstraints()) { + submitTask.setConstraints(new Constraints(key.getField(), Sets.newHashSet())); + } + submitTask.getConstraints().getValues().add(value); + return this; + } + + public Submits addArgs(String arg) { + if (submitTask.getArgs() == null) { + submitTask.setArgs(Lists.newArrayList()); + } + submitTask.getArgs().add(arg); + return this; + } + + public Submits addReadOnlyVolume(String containerPath, String hostPath) { + addVolumes(new Container.Volume(containerPath, hostPath, Container.Volume.DVO.READONLY)); + return this; + } + + public Submits addReadWriteVolume(String containerPath, String hostPath) { + addVolumes(new Container.Volume(containerPath, hostPath, Container.Volume.DVO.READWRITE)); + return this; + } + + public Submits addResources(double cpus, double mems) { + submitTask.setResources(new Resources(Resources.getCpu(cpus), Resources.getMem(mems))); + return this; + } + + private void addVolumes(Container.Volume volume) { + if (submitTask.getContainer().getVolumes() == null) { + List volumes = Lists.newArrayList(); + submitTask.getContainer().setVolumes(volumes); + } + submitTask.getContainer().getVolumes().add(volume); + } + + + @Override + @SuppressWarnings("unchecked") + public Long handle(String requestUrl, String tenantId) { + Result result = null; + try { + result = Restty.create(requestUrl) + .addMediaType(Restty.jsonBody()) + .addHeader(TENANT_ID_HEAD, tenantId) + .requestBody(submitTask) + .post(new ParameterTypeReference>() { + }); + + } catch (IOException e) { + throw new JuiceClientException(ErrorCode.HTTP_REQUEST_ERROR.getCode(), e.getMessage()); + } + return result != null ? result.getData().getTaskId() : null; + } + + + @Data + private class Response { + private Long taskId; + } +} diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/package-info.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/package-info.java new file mode 100644 index 0000000..1136e19 --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/package-info.java @@ -0,0 +1,4 @@ +/** + * Created by xujia on 17/2/12. + */ +package com.hujiang.juice.client.sdk; \ No newline at end of file diff --git a/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/utils/JuiceClient.java b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/utils/JuiceClient.java new file mode 100644 index 0000000..1f4bae8 --- /dev/null +++ b/juice-client-sdk/src/main/java/com/hujiang/juice/client/sdk/utils/JuiceClient.java @@ -0,0 +1,49 @@ +package com.hujiang.juice.client.sdk.utils; + +import com.hujiang.juice.client.sdk.model.*; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +/** + * Created by xujia on 16/12/12. + */ + +@Data +@Slf4j +public class JuiceClient { + + private Operations operations; + private String requestUrl; + private String accessToken; + + private JuiceClient(String requestUrl, String accessToken) { + this.requestUrl = requestUrl; + this.accessToken = accessToken; + } + + private JuiceClient(String requestUrl) { + this.requestUrl = requestUrl; + } + + public JuiceClient setAccessToken(String accessToken) { + this.accessToken = accessToken; + return this; + } + + public static JuiceClient create(String requestUrl, String accessToken) { + return new JuiceClient(requestUrl, accessToken); + } + + public static JuiceClient create(String requestUrl) { + return new JuiceClient(requestUrl); + } + + public JuiceClient setOperations(Operations operations) { + this.operations = operations; + return this; + } + + public T handle() { + return this.operations.handle(requestUrl, accessToken); + } +} diff --git a/juice-common/.gitignore b/juice-common/.gitignore new file mode 100644 index 0000000..ce2fd51 --- /dev/null +++ b/juice-common/.gitignore @@ -0,0 +1,26 @@ +# Created by .ignore support plugin (hsz.mobi) +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### Example user template template +### Example user template + +# IntelliJ project files +.idea/ +.settings/ +target/ +.project +.classpath +*.iml +out +gen diff --git a/juice-common/pom.xml b/juice-common/pom.xml new file mode 100644 index 0000000..39a581f --- /dev/null +++ b/juice-common/pom.xml @@ -0,0 +1,99 @@ + + + + juice + com.hujiang + 1.0-OPEN + + 4.0.0 + + juice-common + + + + org.jetbrains + annotations-java5 + + + commons-codec + commons-codec + + + org.apache.mesos + mesos + + + com.hujiang + juice-jooq + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-core + compile + + + ch.qos.logback + logback-classic + + + org.projectlombok + lombok + + + com.google.guava + guava + + + com.google.code.gson + gson + + + commons-io + commons-io + + + com.squareup.okhttp3 + okhttp + + + redis.clients + jedis + + + org.apache.commons + commons-lang3 + + + javax.servlet + javax.servlet-api + + + com.fasterxml.jackson.core + jackson-databind + + + org.msgpack + msgpack + + + + juice-common + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + + + + + \ No newline at end of file diff --git a/juice-common/src/main/java/com/hujiang/juice/common/config/COMMON.java b/juice-common/src/main/java/com/hujiang/juice/common/config/COMMON.java new file mode 100644 index 0000000..988f7dd --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/config/COMMON.java @@ -0,0 +1,15 @@ +package com.hujiang.juice.common.config; + + +/** + * Created by xujia on 17/1/18. + */ +public class COMMON { + public static final int KILL = 1; + public static final int RECONCILE = 2; + + // task + public static final String CPUS = "cpus"; + public static final String MEMS = "mem"; + +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/config/Configure.java b/juice-common/src/main/java/com/hujiang/juice/common/config/Configure.java new file mode 100644 index 0000000..9d2f05b --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/config/Configure.java @@ -0,0 +1,118 @@ +package com.hujiang.juice.common.config; + +import java.io.FileNotFoundException; +import java.util.*; + +/** + * Created by xujia on 16-6-6. + */ +public class Configure { +// public static final String DEFAULT_CONFIG_NAME_KEY = "config-name"; + + private Map params = new HashMap(); + + private String configName; + + public Configure(String configName) throws FileNotFoundException { + this.configName = configName; + try { + ResourceBundle resourceBundle = ResourceBundle.getBundle(configName); + Enumeration allKeys = resourceBundle.getKeys(); +// params.put(DEFAULT_CONFIG_NAME_KEY, configName); + while (allKeys.hasMoreElements()) { + String key = allKeys.nextElement(); + params.put(key, resourceBundle.getString(key)); + } + } catch (MissingResourceException e) { + throw new FileNotFoundException("configure file [" + configName + "] not found!"); + } + } + + public String getConfigName() { + return configName; + } + + public String getValue(String name) { + if (params.containsKey(name)) { + return params.get(name); + } + return null; + } + public String getValue(String name, String defaultValue) { + if (params.containsKey(name)) { + return params.get(name); + } + return defaultValue; + } + public int getValue(String name, int defaultValue) { + String value = getValue(name); + if (value == null || value.trim().length() == 0) { + return defaultValue; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public long getValue(String name, long defaultValue) { + String value = getValue(name); + if (value == null || value.trim().length() == 0) { + return defaultValue; + } + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public float getValue(String name, float defaultValue) { + String value = getValue(name); + if (value == null || value.trim().length() == 0) { + return defaultValue; + } + try { + return Float.parseFloat(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public double getValue(String name, double defaultValue) { + String value = getValue(name); + if (value == null || value.trim().length() == 0) { + return defaultValue; + } + try { + return Double.parseDouble(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public boolean getValue(String name, boolean defaultValue) { + String value = getValue(name); + if (value == null || value.trim().length() == 0) { + return defaultValue; + } + try { + return Boolean.parseBoolean(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + public Map getParams() { + return params; + } + + public Properties toProperties() { + Properties properties = new Properties(); + for (Map.Entry entry : params.entrySet()) { + properties.setProperty(entry.getKey(), entry.getValue()); + } + return properties; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/error/CommonStatusCode.java b/juice-common/src/main/java/com/hujiang/juice/common/error/CommonStatusCode.java new file mode 100644 index 0000000..b776e68 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/error/CommonStatusCode.java @@ -0,0 +1,79 @@ +package com.hujiang.juice.common.error; + + +import com.hujiang.juice.common.exception.InternalServiceException; +import com.hujiang.juice.common.exception.UnauthorizedException; + +/** + * Created by xujia on 16/12/29. + */ +public enum CommonStatusCode { + SERVICE_OK(0, "success"), + SERVICE_INTERNAL_ERROR(0x81300000,"internal service error"), + NO_PERMISSION(0x81300050, "no permission, auth failed"), + CONSUMED_OUT_CODE(0x81300051, "consumed out error"), + EXPIRED_CODE(0x81300052, "user siqn expired error"), + METHOD_NOT_NULL(0x81300053, "method not null"), + REQUEST_PARAMS_IS_NULL(0x81300054, "request param is null"), + REMOTE_SERVER_RESPONSE_ERROR(0x81300055, "third part remote service not response"), + DATA_ACCESS_ERROR(0x81300056, "db access error"), + NO_RESPONSE_VALUE(0x81300057, "no response"), + RETURN_VALUE_CHECKED_ERROR(0x81300058, "return value not validate"), + QUERY_RECORD_EMPTY(0x81300060, "query error, no record equal in db"), + INVALID_ID_ERRORR(0x81300062, "invalid id error"), + RMQ_PUSH_ERROR(0x81300063, "rabbit mq push value error"), + NO_DATA_ROWS_EXCUTE(0x81300064, "no record in db"), + STAT_NOT_FOUND(0x81300065, "stat not found"), + HEATH_CHECK_ERROR(0x81300066,"health check error"), + INTERFACE_NOT_IMPLEMENT_ERROR(0x81300067,"interface not implement error"), + REQUEST_FOR_REMOTE_SERVICE_ERROR(0x81300068,"request for remote service error"), + OBJECT_INIT_ERROR(0x81300069, "object init error"), + REQUEST_PARAMS_INVALID_ERROR(0x81300070, "request params invalid error"), + CACHE_OPERATING_ERROR(0x81300071,"cache operating error"), + CONFIGURATION_EXCEPTION_ERROR(0x81300072,"configuration exception error"), + REST_EXCEPTION_ERROR(0x81300073,"rest exception error"), + DATABASE_EXCEPTION_ERROR(0x81300074,"database exception error"), + REDIS_OPERATION_ERROR(0x81300075, "redis operation error"), + REDIS_CONNECTION_ERROR(0x81300076,"redis connection error"), + REDIS_CONNECTION_RESOURCE_NOT_NULL(0x81300077,"redis connection resource not null error"); + + + public Integer status; + public String message; + + CommonStatusCode(Integer status, String message) { + this.status = status; + this.message = message; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public UnauthorizedException Error2UnauthorizedException(String message) { + + return new UnauthorizedException(this.status, this.message + ", " + message); + } + + public UnauthorizedException Error2UnauthorizedException() { + + return new UnauthorizedException(this.status, this.message); + } + + public InternalServiceException Error2InternalServiceException() { + + return new InternalServiceException(this.status, this.message); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/error/ErrorCode.java b/juice-common/src/main/java/com/hujiang/juice/common/error/ErrorCode.java new file mode 100644 index 0000000..6569980 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/error/ErrorCode.java @@ -0,0 +1,50 @@ +package com.hujiang.juice.common.error; + +/** + * Created by xujia on 16/9/27. + */ + +public enum ErrorCode { + ENCRYPTION_ERROR(-2127551742, "encryption error"), + SYSTEM_ENV_NOT_VALID(-2127551741, "system env not valid error"), + OBJECT_NOT_EQUAL_ERROR(-2127552256, "object not equal error"), + OBJECT_NOT_NULL_ERROR(-2127552257, "object not null error"), + DATA_ACCESS_ERROR(-2127552258, "data access error"), + NO_DATA_ERROR(-2127552259, "no data error"), + BLANK_VALUE(-2127552260, "string value not blank error"), + VALUE_LENGTH_NOT_EQUAL(-2127552261, "value length not equal error"), + PARSE_OBJECT_ERROR(-2127552262, "parse string to object error"), + + DB_INIT_ERROR(-2127552005, "init db error"), + REDIS_INIT_ERROR(-2127552006, "init db error"), + OBJECT_INIT_ERROR(-2127552007, "init object error"), + HEALTH_CHECK_ERROR(-2127552008, "health check error"), + REDIS_KEY_EXPIRED(-2127552009, "redis key expired"), + REDIS_OPERATION_ERROR(-2127552010, "redis operation error"), + DB_OPERATION_ERROR(-2127552011, "db operation error"), + OBJECT_REFLECT_ERROR(-2127552012, "object reflect error"), + + AUTH_TOKEN_NOT_NULL_ERROR(-2127552013, "auth token not null error"), + AUTH_TOKEN_INVALID_ERROR(-2127552014, "auth token invalid error"), + AUTH_TOKEN_EXPIRED_ERROR(-2127552015, "auth token expired error"), + AUTH_REMOTE_REQUEST_ERROR(-2127552016, "request remote auth service error"), + NOT_SUPPORTED_TYPE_ERROR(-2127552017, "not supported type error"), + HTTP_REQUEST_ERROR(-2127552018, "http request error"), + DATA_FORMAT_ERROR(-2127552019, "data format error"); + + private int code; + private String message; + + ErrorCode(int code, String message) { + this.code = code; + this.message = message; + } + + public int getCode() { + return this.code; + } + + public String getMessage() { + return this.message; + } +} \ No newline at end of file diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/CacheException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/CacheException.java new file mode 100644 index 0000000..8da13ba --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/CacheException.java @@ -0,0 +1,29 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; +import static com.hujiang.juice.common.error.CommonStatusCode.CACHE_OPERATING_ERROR; + +/** + * Created by xujia on 17/2/6. + */ + +public class CacheException extends InternalServiceException { + + public CacheException(int code, String message){ + super(code, message); + } + + public CacheException(String message) { + super(CACHE_OPERATING_ERROR.getStatus(), message); + } + + public CacheException() { + this(CACHE_OPERATING_ERROR); + } + + public CacheException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } + +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/CommonException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/CommonException.java new file mode 100644 index 0000000..a73d5c8 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/CommonException.java @@ -0,0 +1,28 @@ +package com.hujiang.juice.common.exception; + +public class CommonException extends RuntimeException { + + private static final long serialVersionUID = -4426947647526301948L; + + private int code; + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public CommonException(int code, String message) { + super(message); + this.code = code; + } + + @Override + public String toString() { + String s = getClass().getName(); + String message = getLocalizedMessage(); + return "exception code:" + code + "," + ((message != null) ? (s + ": " + message) : s); + } +} \ No newline at end of file diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/ConfigurationException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/ConfigurationException.java new file mode 100644 index 0000000..b5c7f73 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/ConfigurationException.java @@ -0,0 +1,28 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; +import static com.hujiang.juice.common.error.CommonStatusCode.CONFIGURATION_EXCEPTION_ERROR; + +/** + * Created by xujia on 17/2/7. + */ + +public class ConfigurationException extends InternalServiceException { + + public ConfigurationException(int code, String message){ + super(code, message); + } + + public ConfigurationException(String message) { + super(CONFIGURATION_EXCEPTION_ERROR.getStatus(), message); + } + + public ConfigurationException() { + this(CONFIGURATION_EXCEPTION_ERROR); + } + + public ConfigurationException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/DataBaseException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/DataBaseException.java new file mode 100644 index 0000000..65b1dd3 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/DataBaseException.java @@ -0,0 +1,29 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; + +import static com.hujiang.juice.common.error.CommonStatusCode.DATABASE_EXCEPTION_ERROR; + +/** + * Created by xujia on 17/2/7. + */ + +public class DataBaseException extends InternalServiceException { + + public DataBaseException(int code, String message){ + super(code, message); + } + + public DataBaseException(String message) { + super(DATABASE_EXCEPTION_ERROR.getStatus(), message); + } + + public DataBaseException() { + this(DATABASE_EXCEPTION_ERROR); + } + + public DataBaseException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/HealthcheckException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/HealthcheckException.java new file mode 100644 index 0000000..d98e216 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/HealthcheckException.java @@ -0,0 +1,30 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; + +import static com.hujiang.juice.common.error.CommonStatusCode.HEATH_CHECK_ERROR; + +/** + * Created by zhouxiang on 2016/9/6. + */ + +public class HealthcheckException extends CommonException { + private static final long serialVersionUID = 1L; + + public HealthcheckException(int code, String message){ + super(code, message); + } + + public HealthcheckException(String message) { + super(HEATH_CHECK_ERROR.getStatus(), message); + } + + public HealthcheckException() { + this(HEATH_CHECK_ERROR); + } + + public HealthcheckException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/InternalServiceException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/InternalServiceException.java new file mode 100644 index 0000000..6c83b42 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/InternalServiceException.java @@ -0,0 +1,36 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; + +/** + * Created by xujia on 16/2/26. + */ + +public class InternalServiceException extends RuntimeException { + + private static final long serialVersionUID = 1L; + private int code; + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public InternalServiceException(int code, String message) { + super(message); + this.code = code; + } + + public InternalServiceException() { + this(CommonStatusCode.SERVICE_INTERNAL_ERROR); + } + + public InternalServiceException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } + +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/RestException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/RestException.java new file mode 100644 index 0000000..d65d50f --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/RestException.java @@ -0,0 +1,29 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; + +import static com.hujiang.juice.common.error.CommonStatusCode.REST_EXCEPTION_ERROR; + +/** + * Created by xujia on 16/12/5. + */ + +public class RestException extends CommonException { + + public RestException(int code, String message){ + super(code, message); + } + + public RestException(String message) { + super(REST_EXCEPTION_ERROR.getStatus(), message); + } + + public RestException() { + this(REST_EXCEPTION_ERROR); + } + + public RestException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/exception/UnauthorizedException.java b/juice-common/src/main/java/com/hujiang/juice/common/exception/UnauthorizedException.java new file mode 100644 index 0000000..6f99655 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/exception/UnauthorizedException.java @@ -0,0 +1,29 @@ +package com.hujiang.juice.common.exception; + + +import com.hujiang.juice.common.error.CommonStatusCode; +import static com.hujiang.juice.common.error.CommonStatusCode.NO_PERMISSION; + +/** + * Created by xujia on 16/2/26. + */ +public class UnauthorizedException extends CommonException { + + private static final long serialVersionUID = 1L; + + public UnauthorizedException(int code, String message) { + super(code, message); + } + + public UnauthorizedException(String message){ + super(NO_PERMISSION.getStatus(), message); + } + + public UnauthorizedException() { + this(NO_PERMISSION); + } + + public UnauthorizedException(CommonStatusCode statusCode) { + this(statusCode.status, statusCode.message); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Command.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Command.java new file mode 100644 index 0000000..3b76729 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Command.java @@ -0,0 +1,88 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.mesos.v1.Protos; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by xujia on 16/12/2. + */ + +@Data +@Slf4j +public class Command { + + private Environment env; + private List args; + private String value; + + public Command() { + args = new ArrayList<>(); + } + + public Command(String value) { + this.value = value; + } + + public Command(String value, Environment env, List args) { + this.value = value; + this.env = env; + this.args = args; + } + + public Command(Environment env) { + this.env = env; + args = new ArrayList<>(); + } + + public Command(Environment env, List args) { + this.env = env; + this.args = args; + } + + public void setEnv(Protos.CommandInfo.Builder builder) { + if (null != env) { + builder.setEnvironment(Protos.Environment.newBuilder() + .addVariables(Protos.Environment.Variable.newBuilder() + .setName(env.getName()) + .setValue(env.getValue())) + .build()); + } + } + public @NotNull Protos.CommandInfo protos(boolean isShell) { + + Protos.CommandInfo.Builder builder = Protos.CommandInfo.newBuilder(); + if(isShell) { + builder.setValue(value); + } else { + builder.setShell(false); + if (null != args && !args.isEmpty()) { + builder.addAllArguments(args); + } + } + + setEnv(builder); + + return builder.build(); + } + + + public static Environment newEnvironment(String name, String value) { + return new Environment(name, value); + } + + @Data + public static class Environment { + private String name; + private String value; + + public Environment(String name, String value) { + this.name = name; + this.value = value; + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Constraints.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Constraints.java new file mode 100644 index 0000000..45c5f2b --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Constraints.java @@ -0,0 +1,44 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; + +import java.util.Map; +import java.util.Set; + +/** + * Created by xujia on 17/3/3. + */ +@Data +public class Constraints { + + private String field; + private Set values; + + public Constraints(String field, Set values) { + this.field = field; + this.values = values; + } + + public enum FIELD { + RACK_ID("rack_id"), + HOSTNAME("hostname"); + + private String field; + + public String getField() { + return field; + } + + FIELD(String field) { + this.field = field; + } + } + + public boolean isAvailable(Map> facts) { + + Set fValues = facts.get(field); + return values.stream().parallel().anyMatch(fValues::contains); + } +} + + diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Container.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Container.java new file mode 100644 index 0000000..5570ff3 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Container.java @@ -0,0 +1,99 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.mesos.v1.Protos; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +import static com.hujiang.juice.common.model.Container.Volume.DVO.*; +import static com.hujiang.juice.common.model.Container.TYPE.*; + +/** + * Created by xujia on 16/12/2. + */ +@Data +@Slf4j +public class Container { + + private Docker docker; + private String type; + private List volumes; + + public Container(Docker docker) { + this.docker = docker; + this.type = TYPE.DOCKER; + } + + public Container(Docker docker, List volumes) { + this.type = TYPE.DOCKER; + this.docker = docker; + this.volumes = volumes; + } + + public Container(Docker docker, String type, List volumes) { + this.type = type; + this.docker = docker; + this.volumes = volumes; + } + + public @NotNull Protos.ContainerInfo protos() { + + Protos.ContainerInfo.Builder builder + = Protos.ContainerInfo.newBuilder() + .setDocker(docker.protos()) + .setType(getType()); + + if(null != volumes && !volumes.isEmpty()) { + volumes.forEach( + volume -> { + builder.addVolumes(Protos.Volume.newBuilder() + .setContainerPath(volume.getContainerPath()) + .setHostPath(volume.getHostPath()) + .setMode(volume.getMode()) + .build()); + } + ); + } + + return builder.build(); + } + + @Data + public static class Volume { + private String containerPath; + private String hostPath; + private String dvo; + + public Volume(String containerPath, String hostPath, String dvo) { + this.containerPath = containerPath; + this.hostPath = hostPath; + this.dvo = dvo; + } + + public Protos.Volume.Mode getMode() { + switch (dvo) { + case READONLY : return Protos.Volume.Mode.RO; + default: return Protos.Volume.Mode.RW; + } + } + + public interface DVO { + String READONLY = "RO"; + String READWRITE = "RW"; + } + } + + public Protos.ContainerInfo.Type getType() { + switch (type) { + case MESOS : return Protos.ContainerInfo.Type.MESOS; + default: return Protos.ContainerInfo.Type.DOCKER; + } + } + + public interface TYPE { + String DOCKER = "DOCKER"; + String MESOS = "MESOS"; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Docker.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Docker.java new file mode 100644 index 0000000..fca42b5 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Docker.java @@ -0,0 +1,145 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; +import org.apache.mesos.v1.Protos; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +import static com.hujiang.juice.common.model.Docker.NetWork.*; + +/** + * Created by xujia on 17/1/17. + */ + +@Data +public class Docker { + private String image; + private Boolean forcePullImage; + private Boolean privileged; + private String net; + private List parameters; + private List portMappings; + + + public Docker(String image){ + this.image = image; + forcePullImage = true; + privileged = false; + net = BRIDGE; + parameters = new ArrayList<>(); + portMappings = new ArrayList<>(); + + } + + public Docker(String image, boolean forcePullImage){ + this.image = image; + this.forcePullImage = forcePullImage; + privileged = false; + net = BRIDGE; + parameters = new ArrayList<>(); + portMappings = new ArrayList<>(); + } + + public Docker(String image, boolean forcePullImage, String net){ + this.image = image; + this.forcePullImage = forcePullImage; + this.net = net; + privileged = false; + parameters = new ArrayList<>(); + portMappings = new ArrayList<>(); + } + + public Docker(String image, boolean forcePullImage, boolean privileged, String net, List parameters, List portMappings) { + this.image = image; + this.forcePullImage = forcePullImage; + this.privileged = privileged; + this.net = net; + this.parameters = parameters; + this.portMappings = portMappings; + } + + public @NotNull + Protos.ContainerInfo.DockerInfo protos() { + + Protos.ContainerInfo.DockerInfo.Builder dockerBuild = Protos.ContainerInfo.DockerInfo.newBuilder() + .setImage(image) + .setForcePullImage(forcePullImage) + .setPrivileged(privileged) + .setNetwork(exchange()); + + + if (null != parameters) { + parameters.forEach( + par -> { + dockerBuild.addParameters(Protos.Parameter.newBuilder().setKey(par.getKey()).setValue(par.getValue())); + } + ); + } + if (null != portMappings) { + portMappings.forEach( + por -> { + dockerBuild.addPortMappings(Protos.ContainerInfo.DockerInfo.PortMapping.newBuilder() + .setContainerPort(por.getContainerPort()) + .setHostPort(por.getHostPort()) + .setProtocol(por.getProtocol())); + } + ); + } + return dockerBuild.build(); + } + + public static PortMapping newPortMapping(int containerPort, int hostPort, String protocol) { + return new PortMapping(containerPort, hostPort, protocol); + } + + public static Parameter newParameter(String key, String value) { + return new Parameter(key, value); + } + + public static Container.Volume newVolume(String containerPath, String hostPath, String dvo) { + return new Container.Volume(containerPath, hostPath, dvo); + } + + private Protos.ContainerInfo.DockerInfo.Network exchange() { + if(null == net) { + return Protos.ContainerInfo.DockerInfo.Network.BRIDGE; + } + switch (net) { + case HOST: return Protos.ContainerInfo.DockerInfo.Network.HOST; + case NONE: return Protos.ContainerInfo.DockerInfo.Network.NONE; + case USER: return Protos.ContainerInfo.DockerInfo.Network.USER; + default: return Protos.ContainerInfo.DockerInfo.Network.BRIDGE; + } + } + + public interface NetWork { + String BRIDGE = "BRIDGE"; + String HOST = "HOST"; + String NONE = "NONE"; + String USER = "USER"; + } + + @Data + public static class PortMapping { + private int containerPort; + private int hostPort; + private String protocol; + public PortMapping(int containerPort, int hostPort, String protocol) { + this.containerPort = containerPort; + this.hostPort = hostPort; + this.protocol = protocol; + } + } + + @Data + public static class Parameter { + private String key; + private String value; + public Parameter(String key, String value) { + this.key = key; + this.value = value; + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Expire.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Expire.java new file mode 100644 index 0000000..af9733a --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Expire.java @@ -0,0 +1,27 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; + +/** + * Created by xujia on 17/3/7. + */ + +@Data +public class Expire { + private long firstReservedTimes; + private int resourceLack; + private int offerLack; + public Expire() { + firstReservedTimes = 0; + resourceLack = 0; + offerLack = 0; + } + + public void incrementResourceLack() { + resourceLack++; + } + + public void incrementOfferLack() { + offerLack++; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Resources.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Resources.java new file mode 100644 index 0000000..d27eb96 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Resources.java @@ -0,0 +1,123 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.apache.mesos.v1.Protos; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +import static com.hujiang.juice.common.config.COMMON.CPUS; +import static com.hujiang.juice.common.config.COMMON.MEMS; + + +/** + * Created by xujia on 16/12/2. + */ + +@Data +public class Resources { + + private String role; + private double cpu; + private double mem; + + public Resources(double cpu, double mem) { + this.cpu = cpu; + this.mem = mem; + this.role = "*"; + } + + public Resources(double cpu, double mem, String role) { + this.cpu = cpu; + this.mem = mem; + this.role = role; + } + + private + @NotNull + Protos.Resource addResource(@NotNull String name, @NotNull double value) { + return Protos.Resource.newBuilder() + .setName(name) + .setRole(role) + .setType(Protos.Value.Type.SCALAR) + .setScalar(Protos.Value.Scalar.newBuilder().setValue(value).build()) + .build(); + } + + public + @NotNull + List protos() { + List resources = new ArrayList<>(); + resources.add(addResource(CPUS, cpu)); + resources.add(addResource(MEMS, mem)); + return resources; + } + + public void checkSet() { + if(StringUtils.isBlank(role)) { + role = "*"; + } + cpu = getCpu(cpu); + mem = getMem(mem); + } + + public static double getCpu(double v) { + + if (v >= CPU_LEVEL.MAX.getLevel()) { + return CPU_LEVEL.MAX.getLevel(); + } + + if (v <= CPU_LEVEL.MIN.getLevel()) { + return CPU_LEVEL.MIN.getLevel(); + } + + return v; + } + + public static double getMem(double v) { + + if (v >= MEM_LEVEL.MAX.getLevel()) { + return MEM_LEVEL.MAX.getLevel(); + } + + if (v <= MEM_LEVEL.MIN.getLevel()) { + return MEM_LEVEL.MIN.getLevel(); + } + + return v; + } + + public enum CPU_LEVEL { + MIN(0.1), + MAX(16.0), + DEFAULT(1.0); + + private double level; + + CPU_LEVEL(double level) { + this.level = level; + } + + public double getLevel() { + return level; + } + } + + public enum MEM_LEVEL { + MIN(256.0), + MAX(8192.0 * 2), + DEFAULT(512.0); + + private double level; + + MEM_LEVEL(double level) { + this.level = level; + } + + public double getLevel() { + return level; + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/Task.java b/juice-common/src/main/java/com/hujiang/juice/common/model/Task.java new file mode 100644 index 0000000..cecc19e --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/Task.java @@ -0,0 +1,85 @@ +package com.hujiang.juice.common.model; + +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.exception.CommonException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.mesos.v1.Protos; +import org.jetbrains.annotations.NotNull; + +/** + * Created by xujia on 16/11/30. + */ + +@Slf4j +@Data +public class Task { + + private Resources resources; + private Container container; + private Command command; + private Constraints constraints; + + private String taskName; + private long taskId; + private Expire expire; + + public Task(Resources resources, Container container, Command command, Constraints constraints, String taskName) { + this.resources = resources; + this.container = container; + this.command = command; + this.taskName = taskName; + this.constraints = constraints; + this.expire = new Expire(); + } + + public Task(Resources resources, Container container, Command command, Constraints constraints, String taskName, long taskId) { + this.resources = resources; + this.container = container; + this.command = command; + this.taskName = taskName; + this.taskId = taskId; + this.constraints = constraints; + this.expire = new Expire(); + } + + public @NotNull static long splitTaskNameId(String value) { + + if(!StringUtils.isNotBlank(value)) { + log.warn("taskId is null!"); + throw new CommonException(ErrorCode.OBJECT_NOT_NULL_ERROR.getCode(), "taskId is null!"); + } + + String[] parts = value.split("-"); + if(parts.length != 2) { + log.warn("value length not reach min parts!"); + throw new CommonException(ErrorCode.VALUE_LENGTH_NOT_EQUAL.getCode(), "value length not reach min parts!"); + } + return Long.parseLong(parts[1]); + } + + public @NotNull static String generateTaskNameId(String name, long id) { + StringBuilder sb = new StringBuilder(); + return sb.append(name.replaceAll("-", "_")).append("-").append(id).toString(); + } + + public @NotNull + Protos.TaskInfo getTask( + final @NotNull Protos.AgentID agentID + ) { + Protos.TaskInfo.Builder taskInfoBuilder = Protos.TaskInfo.newBuilder(); + taskInfoBuilder.setName(taskName); + taskInfoBuilder.setTaskId(Protos.TaskID.newBuilder().setValue(generateTaskNameId(taskName, taskId))); + taskInfoBuilder.setAgentId(agentID); + taskInfoBuilder.addAllResources(resources.protos()); + + if(null != command && StringUtils.isNotBlank(command.getValue())) { + // run shell + return taskInfoBuilder.setCommand(command.protos(true)).build(); + } else { + // run docker + return taskInfoBuilder.setContainer(container.protos()).setCommand(command.protos(false)).build(); + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/model/TaskManagement.java b/juice-common/src/main/java/com/hujiang/juice/common/model/TaskManagement.java new file mode 100644 index 0000000..6918060 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/model/TaskManagement.java @@ -0,0 +1,59 @@ +package com.hujiang.juice.common.model; + +import lombok.Data; + +import java.util.List; + +import static java.lang.System.currentTimeMillis; +import static org.apache.mesos.v1.Protos.*; +/** + * Created by xujia on 17/1/18. + */ + +@Data +public class TaskManagement { + List taskAgentRels; + long retries; + int type; + + public TaskManagement(List taskAgentRels, int type) { + this.taskAgentRels = taskAgentRels; + this.type = type; + this.retries = currentTimeMillis(); + } + + public static TaskAgentRel newTaskAgentRel(long taskId, String taskName, String agentId) { + return new TaskAgentRel(taskId, taskName, agentId); + } + + @Data + public static class TaskAgentRel { + long taskId; + String taskName; + String agentId; + + public TaskAgentRel(long taskId, String taskName, String agentId) { + this.taskId = taskId; + this.taskName = taskName; + this.agentId = agentId; + } + + @Override + public String toString() { + return "taskId : " + taskId + ", taskName : " + taskName + ", agentId : " + agentId; + } + + public TaskID protosTaskId() { + return TaskID.newBuilder().setValue(Task.generateTaskNameId(taskName, taskId)).build(); + } + + public AgentID protosAgentId() { + return AgentID.newBuilder().setValue(agentId).build(); + } + + public void setAgentId(String agentId){ + this.agentId = agentId; + } + } + +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/package-info.java b/juice-common/src/main/java/com/hujiang/juice/common/package-info.java new file mode 100644 index 0000000..2a8ee23 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/package-info.java @@ -0,0 +1,4 @@ +/** + * Created by xujia on 16/12/5. + */ +package com.hujiang.juice.common; \ No newline at end of file diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/BeanUtil.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/BeanUtil.java new file mode 100644 index 0000000..d30c705 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/BeanUtil.java @@ -0,0 +1,87 @@ +package com.hujiang.juice.common.utils; + +import lombok.extern.slf4j.Slf4j; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Created by xujia on 17-3-22. + */ +@Slf4j +public class BeanUtil { + + public static Map obj2Map(Class tClass, T t) { + return obj2Map(tClass, t, true); + } + + public static Map obj2Map(Class tClass, T t, boolean isFilterNull) { + Field[] fields = tClass.getDeclaredFields(); + List fieldNameList = Arrays.stream(fields) + .filter(f -> !f.getName().equalsIgnoreCase("serialVersionUID")) + .map(f -> f.getName()) + .collect(Collectors.toList()); + Map map = new HashMap(); + fieldNameList.forEach(field -> { + String methodName = "get" + field.substring(0, 1).toUpperCase() + field.substring(1); + Method method = null; + try { + method = tClass.getMethod(methodName); + } catch (Exception e) { + log.info(e.getMessage(), e); + } + if (method != null) { + try { + Object obj = method.invoke(t); + if (!isFilterNull || obj != null) { + map.put(field, obj); + } + } catch (Exception e) { + log.info(e.getMessage(), e); + } + } + }); + return map; + } + + public static T map2Obj(Class tClass, Map map) throws Exception { + return map2Obj(tClass, map, false); + } + + public static T map2Obj(Class tClass, Map map, boolean isFilterNull) throws Exception { + T t = null; + try { + t = tClass.newInstance(); + } catch (Exception e) { + log.warn(e.getMessage(), e); + throw new Exception("create Obect[" + tClass.getName() + "] fail.", e); + } + final T tmp = t; + map.entrySet().forEach(entry -> { + String field = entry.getKey(); + String methodName = "set" + field.substring(0, 1).toUpperCase() + field.substring(1); + Method method = null; + try { + method = tClass.getMethod(methodName); + } catch (Exception e) { + log.info(e.getMessage(), e); + } + if (method != null) { + Object value = entry.getValue(); + try { + if (!isFilterNull || value != null) { + method.invoke(tmp, value); + } + } catch (Exception e) { + log.info(e.getMessage(), e); + } + } + }); + return t; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/CommonUtils.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/CommonUtils.java new file mode 100644 index 0000000..1bc19e0 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/CommonUtils.java @@ -0,0 +1,17 @@ +package com.hujiang.juice.common.utils; + +/** + * Created by xujia on 16/11/21. + */ +public class CommonUtils { + + public static String fixUrl(String url) { + if (!url.startsWith("http://")) url = "http://" + url; + if (url.endsWith("/")) url = url.substring(0, url.length() - 1); + return url; + } + + public static long currentTimeSeconds() { + return System.currentTimeMillis() / 1000; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/DESUtil.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/DESUtil.java new file mode 100644 index 0000000..4157582 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/DESUtil.java @@ -0,0 +1,103 @@ +package com.hujiang.juice.common.utils; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESKeySpec; +import java.security.SecureRandom; +import java.util.Base64; + +/** + * Created by xujia on 2016/8/24. + */ +public class DESUtil { + + public static final byte[] desKey = new byte[]{21, 1, -110, 82, -32, -85, -128, -65}; + + /** + * 数据加密，算法（DES） + * + * @param data + * 要进行加密的数据 + * @param desKey DES密钥 + * @return 加密后的数据 + */ + public static String encrypt(String data, byte[] desKey) { + String encryptedData = null; + try { + // DES算法要求有一个可信任的随机数源 + SecureRandom sr = new SecureRandom(); + DESKeySpec deskey = new DESKeySpec(desKey); + // 创建一个密匙工厂，然后用它把DESKeySpec转换成一个SecretKey对象 + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); + SecretKey key = keyFactory.generateSecret(deskey); + // 加密对象 + Cipher cipher = Cipher.getInstance("DES"); + cipher.init(Cipher.ENCRYPT_MODE, key, sr); + // 加密，并把字节数组编码成字符串 + encryptedData = Base64.getEncoder().encodeToString(cipher.doFinal(data.getBytes())); + } catch (Exception e) { +// log.error("加密错误，错误信息：", e); + throw new RuntimeException("加密错误，错误信息：", e); + } + return encryptedData; + } + + /** + * 向前兼容，key固定 + * @deprecated + * @param data 要加密的数据 + * @return 加密后的数据 + */ + public static String encrypt(String data){ + return encrypt(data, desKey); + } + + /** + * 数据解密，算法（DES） + * + * @param cryptData + * 加密数据 + * @param desKey DES密钥 + * @return 解密后的数据 + */ + public static String decrypt(String cryptData, byte[] desKey) { + String decryptedData = null; + try { + // DES算法要求有一个可信任的随机数源 + SecureRandom sr = new SecureRandom(); + DESKeySpec deskey = new DESKeySpec(desKey); + // 创建一个密匙工厂，然后用它把DESKeySpec转换成一个SecretKey对象 + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); + SecretKey key = keyFactory.generateSecret(deskey); + // 解密对象 + Cipher cipher = Cipher.getInstance("DES"); + cipher.init(Cipher.DECRYPT_MODE, key, sr); + // 把字符串解码为字节数组，并解密 + decryptedData = new String(cipher.doFinal(Base64.getDecoder().decode(cryptData))); + } catch (Exception e) { +// log.error("解密错误，错误信息：", e); + throw new RuntimeException("解密错误，错误信息：", e); + } + return decryptedData; + } + + /** + * 向前兼容，key固定 + * @deprecated + * @param cryptData 加密数据 + * @return 解密后的数据 + */ + public static String decrypt(String cryptData){ + return decrypt(cryptData, desKey); + } + + + public static void main(String[] args){ + String encryptString= encrypt("9$r05YT*6G", desKey); + System.out.println(encryptString); + + String desencryptString = decrypt(encryptString, desKey); + System.out.println(desencryptString); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/JsonUtils.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/JsonUtils.java new file mode 100644 index 0000000..51311bb --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/JsonUtils.java @@ -0,0 +1,21 @@ +package com.hujiang.juice.common.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Created by xujia on 2016/6/14. + */ +public class JsonUtils { + + private static final String MediaType = "application/json;charset=UTF-8"; + private static ObjectMapper mapper = new ObjectMapper(); + + public static void writeJsonResult(HttpServletResponse response, Object data) throws IOException { + response.setContentType(MediaType); + mapper.writeValue(response.getOutputStream(), data); + } + +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/CacheUtils.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/CacheUtils.java new file mode 100644 index 0000000..0d548dd --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/CacheUtils.java @@ -0,0 +1,15 @@ +package com.hujiang.juice.common.utils.cache; + +import org.jetbrains.annotations.NotNull; + +/** + * Created by xujia on 17/3/9. + */ +public interface CacheUtils { + long pushToQueue(@NotNull String queue, @NotNull String value); + String popFromQueue(@NotNull String queue); + long lengthOfQueue(@NotNull String queue); + boolean setExpired(String key, String value, int expired); + boolean existsKey(String key); + long delete(String key); +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisCacheUtils.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisCacheUtils.java new file mode 100644 index 0000000..877e6a3 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisCacheUtils.java @@ -0,0 +1,106 @@ +package com.hujiang.juice.common.utils.cache; + +import com.hujiang.juice.common.exception.CacheException; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + + +/** + * Created by xujia on 17/1/19. + */ + +@Slf4j +@Data +public class RedisCacheUtils implements CacheUtils{ + + private RedisUtil redisUtil; + + public RedisCacheUtils(RedisUtil redisUtil) { + this.redisUtil = redisUtil; + } + + public long pushToQueue(@NotNull String queue, @NotNull String value){ + try { + return push(queue, value); + } catch (CacheException e) { + String error = "cache operation error, push to " + queue + " failed, Task :" + value; + log.error(error); + throw e; + } + } + + public String popFromQueue(@NotNull String queue){ + try { + return pop(queue); + } catch (CacheException e) { + String error = "cache operation error, pop from " + queue + " failed"; + log.error(error); + throw e; + } + } + + public long lengthOfQueue(@NotNull String queue){ + try { + return length(queue); + } catch (CacheException e) { + String error = "cache operation error, get length" + queue + " failed"; + log.error(error); + throw e; + } + } + + public boolean setExpired(String key, String value, int expired) { + try { + return setex(key, value, expired); + } catch (CacheException e) { + String error = "cache setex operation error, key : " + key + "， value ：" + value + ", expired : " + expired; + log.error(error); + throw e; + } + } + + public boolean existsKey(String key){ + try { + return exists(key); + } catch (CacheException e) { + String error = "cache setex operation error, key : " + key; + log.error(error); + throw e; + } + } + + public long delete(String key) { + try { + return del(key); + } catch (CacheException e) { + String error = "cache delete operation error, key : " + key; + log.error(error); + throw e; + } + } + + private boolean setex(String key, String value, int expireds) { + return redisUtil.setex(key, value, expireds); + } + + private boolean exists(String key) { + return redisUtil.exists(key); + } + + private long del(String key){ + return redisUtil.del(key); + } + + private Long push(String lname, String value) { + return redisUtil.lPush(lname, value); + } + + private String pop(String lname) { + return redisUtil.rPop(lname); + } + + private long length(String lname) { + return redisUtil.llen(lname); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisUtil.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisUtil.java new file mode 100644 index 0000000..30af243 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/cache/RedisUtil.java @@ -0,0 +1,454 @@ +package com.hujiang.juice.common.utils.cache; + + +import com.hujiang.juice.common.error.CommonStatusCode; +import com.hujiang.juice.common.exception.CacheException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import redis.clients.jedis.*; +import redis.clients.jedis.exceptions.JedisConnectionException; + +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Created by zhouxiang on 2016/8/1. + */ +@Slf4j +public class RedisUtil { + private JedisPool pool = null; + + public Jedis getResource() { + return pool.getResource(); + } + + /** + *

传入ip和端口号构建redis 连接池

+ * @param ip ip + * @param prot 端口 + */ + public RedisUtil(String ip, int prot) { + if (pool == null) { + JedisPoolConfig config = new JedisPoolConfig(); + config.setMaxTotal(500); + config.setMaxIdle(5); + config.setMaxWaitMillis(3000); + config.setTestOnBorrow(true); + pool = new JedisPool(config, ip, prot, 5000); + } + } + + + /** + *

传入ip和端口号构建redis 连接池

+ * @param ip ip + * @param prot 端口 + */ + public RedisUtil(String ip, int prot, String password) { + if (pool == null) { + JedisPoolConfig config = new JedisPoolConfig(); + config.setMaxTotal(500); + config.setMaxIdle(5); + config.setMaxWaitMillis(3000); + config.setTestOnBorrow(true); + if(StringUtils.isNotBlank(password)) + pool = new JedisPool(config, ip, prot, 5000, password); + else + pool = new JedisPool(config, ip, prot, 5000); + } + } + + /** + *

通过配置对象 ip 端口 构建连接池

+ * @param config 配置对象 + * @param ip ip + * @param prot 端口 + */ + public RedisUtil(JedisPoolConfig config , String ip, int prot){ + if (pool == null) { + pool = new JedisPool(config,ip,prot,5000); + } + } + + /** + *

通过配置对象 ip 端口 超时时间 构建连接池

+ * @param config 配置对象 + * @param ip ip + * @param prot 端口 + * @param timeout 超时时间 + */ + public RedisUtil(JedisPoolConfig config , String ip, int prot , int timeout){ + if (pool == null) { + pool = new JedisPool(config,ip,prot,timeout); + } + } + + /** + *

通过连接池对象 构建一个连接池

+ * @param pool 连接池对象 + */ + public RedisUtil(JedisPool pool){ + if (this.pool == null) { + this.pool = pool; + } + } + + /** + *

通过key获取储存在redis中的value

+ *

并释放连接

+ * @param key + * @return 成功返回value 失败返回null + */ + public String get(String key){ + Jedis jedis = null; + try{ + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.get(key); + } catch (Exception e) { + log.error("redis set error, key : " + key); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

向redis存入key和value,并释放连接资源

+ *

如果key已经存在 则覆盖

+ * @param key + * @param value + * @return 成功 返回OK 失败返回 0 + */ + public String set(String key,String value){ + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.set(key, value); + } catch (Exception e) { + log.error("redis set error, key : " + key + " value : " + value); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + + /** + *

删除指定的key,也可以传入一个包含key的数组

+ * @param keys 一个key 也可以使 string 数组 + * @return 返回删除成功的个数 + */ + public Long del(String...keys){ + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.del(keys); + } catch (Exception e) { + log.error("redis set error, keys : " + Arrays.stream(keys).collect(Collectors.joining(","))); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

通过key向指定的value值追加值

+ * @param key + * @param str + * @return 成功返回 添加后value的长度 失败 返回 添加的 value 的长度 异常返回0L + */ + public Long append(String key ,String str){ + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.append(key, str); + } catch (Exception e) { + log.error("redis set error, key : " + key + " value : " + str); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

判断key是否存在

+ * @param key + * @return true OR false + */ + public Boolean exists(String key){ + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.exists(key); + } catch (Exception e) { + log.error("redis set error, key : " + key); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + public Boolean setex(String key, String value, int expireTime) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return null != jedis.setex(key, expireTime, value); + } catch (Exception e) { + log.error("redis set error, key : " + key + " value : " + value + " expireTime : " + expireTime); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

队列左插入一个value

+ * @param listName 队列名 + * value 元素值 + * @return 插入后队列的元素个数 + */ + public Long lPush(String listName, String value) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.lpush(listName, value); + } catch (Exception e) { + log.error("redis lPush error, ListName : " + listName + " value : " + value); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

队列右弹出一个元素

+ * @param listName 队列名 + * + * @return 元素值 + */ + public String rPop(String listName) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.rpop(listName); + } catch (Exception e) { + log.error("redis rPop error, ListName : " + listName); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

队列元素个数

+ * @param listName 队列名 + * + * @return 元素个数 + */ + public Long llen(String listName) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.llen(listName); + } catch (Exception e) { + log.error("redis llen error, ListName : " + listName); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

在指定的Set中插入一个元素

+ * @param setName set名 + * value + * @return Set中的元素个数 + */ + public Long sadd(String setName, String value) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.sadd(setName, value); + } catch (Exception e) { + log.error("redis sadd error, setName : " + setName + " value : " + value); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

在指定的Set中删除一个元素

+ * @param setName set名 + * value + * @return Set中的元素个数 + */ + public Long srem(String setName, String value) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.srem(setName, value); + } catch (Exception e) { + log.error("redis srem error, setName : " + setName + " value : " + value); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + /** + *

返回Set中的所有元素

+ * @param setName set名 + * value + * @return 元素值 + */ + public Set smembers(String setName) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.smembers(setName); + } catch (Exception e) { + log.error("redis smembers error, setName : " + setName); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + public void subscribe(JedisPubSub subscriber, String channel) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + jedis.subscribe(subscriber, channel); + } catch (Exception e) { + log.error("redis subscribe error, channel : " + channel); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + public Long publish(String channel, String message) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.publish(channel, message); + } catch (Exception e) { + log.error("redis publish error, channel : " + channel + ", message : " + message); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + + public boolean unLock(String key) { + Jedis jedis = null; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + return jedis.del(key) > 0; + } catch (Exception e) { + log.error("set redis unlock error!"); + throw throwCacheException(e); + } finally { + returnResource(jedis); + } + } + + public boolean lock(String key, int seconds, String value) { + Jedis jedis = null; + Transaction tx; + try { + jedis = pool.getResource(); + if(jedis == null) { + throw new CacheException(CommonStatusCode.REDIS_CONNECTION_RESOURCE_NOT_NULL); + } + + jedis.watch(key); + if (!jedis.exists(key)) { + tx = jedis.multi(); + tx.setex(key, seconds, value); + if (null != tx.exec()) { + return true; + } + } + return false; + } catch (Exception e) { + log.error("set redis lock key error!"); + throw throwCacheException(e); + } finally { + if (null != jedis) { + jedis.unwatch(); + } + returnResource(jedis); + } + } + + private CacheException throwCacheException(Exception e) { + log.error("error due to : " + e.getMessage()); + if(e instanceof JedisConnectionException) { + return new CacheException(CommonStatusCode.REDIS_CONNECTION_ERROR.getStatus(), "jedis connection exception"); + } else if(e instanceof CacheException) { + return (CacheException)e; + } + return new CacheException(CommonStatusCode.REDIS_OPERATION_ERROR.getStatus(), e.getMessage()); + } + + /** + * 返还到连接池 + * + * @param jedis + */ + public static void returnResource(Jedis jedis) { + if (jedis != null) { + jedis.close(); + } + } + +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/db/DaoUtils.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/db/DaoUtils.java new file mode 100644 index 0000000..f56531e --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/db/DaoUtils.java @@ -0,0 +1,172 @@ +package com.hujiang.juice.common.utils.db; + +import com.hujiang.jooq.juice.tables.pojos.JuiceFramework; +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.common.exception.DataBaseException; +import lombok.extern.slf4j.Slf4j; +import org.jooq.Configuration; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; + +import java.util.List; + +import static com.hujiang.juice.common.error.ErrorCode.DB_OPERATION_ERROR; + +/** + * Created by xujia on 17/1/18. + */ + +@Slf4j +public class DaoUtils { + + private JuiceDao juiceDao; + + public DaoUtils(JuiceDao juiceDao) { + this.juiceDao = juiceDao; + } + + public DSLContext getContext() { + return juiceDao.getContext(); + } + + public JuiceTask queryTask(long taskId) { + try { + return juiceDao.query(taskId); + } catch (Exception e) { + String error = "query task error, taskId : " + taskId + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public JuiceTask queryTask(String tenantId, long taskId) { + try { + return juiceDao.query(tenantId, taskId); + } catch (Exception e) { + String error = "query task error, taskId : " + taskId + ", tenantId : " + tenantId + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public JuiceTask queryRunningTask(String tenantId, long taskId) { + try { + return juiceDao.queryRunningTask(tenantId, taskId); + } catch (Exception e) { + String error = "query running task error, taskId : " + taskId + ", tenantId : " + tenantId + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public List queryRunningTasks(String tenantId, List taskIds) { + try { + return juiceDao.queryRunningTasks(tenantId, taskIds); + } catch (Exception e) { + String error = "query running tasks error, due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public List queryTasks(String tenantId, List taskIds) { + try { + return juiceDao.queryTasks(tenantId, taskIds); + } catch (Exception e) { + String error = "query running tasks error, due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean finishTask(long taskId, byte type, String message) { + try { + return juiceDao.finish(taskId, type, message); + } catch (Exception e) { + String error = "finish task error, taskId : " + taskId + ", type : " + type + ", message : " + message + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean finishTaskWithCallBack(long taskId, byte type, String message) { + try { + return juiceDao.finishWithCallBack(taskId, type, message); + } catch (Exception e) { + String error = "finish task error, taskId : " + taskId + ", type : " + type + ", message : " + message + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean expiredTask(long taskId, int expiredHours) { + try { + return juiceDao.isExpired(taskId, expiredHours); + } catch (Exception e) { + String error = "expired task error, taskId : " + taskId + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean updateTask(long taskId, byte type, String message) { + try { + return juiceDao.update(taskId, type, message); + } catch (Exception e) { + String error = "update task error, taskId : " + taskId + ", type : " + type + ", message : " + message + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean updateTask(long taskId, String agent) { + try { + return juiceDao.update(taskId, agent); + } catch (Exception e) { + String warn = "update task error, taskId : " + taskId + ", agent : " + agent + ", due to : " + e.getMessage(); + log.warn(warn); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean submit(Configuration configuration, long taskId, String tenantId, String callbackUrl, String taskName, String dockerName, String commands) { + try { + return juiceDao.submit(DSL.using(configuration), taskId, tenantId, callbackUrl, taskName, dockerName, commands); + } catch (Exception e) { + String error = "submit task error, taskId : " + taskId + ", tenantId : " + tenantId + ", taskName : " + taskName + ", callbackUrl : " + callbackUrl + ", dockerName : " + dockerName + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + + public boolean saveFrameworkId(String frameworkTag, String frameworkId) { + try { + return juiceDao.saveFrameworkId(frameworkTag, frameworkId); + } catch (Exception e) { + String error = "refreshFramework error, frameworkTag : " + frameworkTag + ", frameworkId : " + frameworkId + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public JuiceFramework queryFramework(String frameworkTag) { + try { + return juiceDao.queryFramework(frameworkTag); + } catch (Exception e) { + String error = "queryFramework error, frameworkTag : " + frameworkTag + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } + + public boolean unActiveFramework(String frameworkTag) { + try { + return juiceDao.unActiveFramework(frameworkTag); + } catch (Exception e) { + String error = "unActiveFramework error, frameworkTag : " + frameworkTag + ", due to : " + e.getMessage(); + log.error(error); + throw new DataBaseException(DB_OPERATION_ERROR.getCode(), e.getMessage()); + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/db/JuiceDao.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/db/JuiceDao.java new file mode 100644 index 0000000..a92bf21 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/db/JuiceDao.java @@ -0,0 +1,165 @@ +package com.hujiang.juice.common.utils.db; + + +import com.hujiang.jooq.juice.tables.pojos.JuiceFramework; +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.common.vo.TaskResult; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.jooq.types.DayToSecond; + +import java.time.Duration; +import java.util.List; + +import static com.hujiang.jooq.juice.tables.JuiceFramework.JUICE_FRAMEWORK; +import static com.hujiang.jooq.juice.tables.JuiceTask.JUICE_TASK; +import static org.jooq.impl.DSL.currentTimestamp; + + +/** + * Created by xujia on 16/12/5. + */ + +@Slf4j +@Data +public class JuiceDao { + + private DSLContext context; + private static final int EXPIRE_DURATION = 24; + + public JuiceDao(DSLContext dlsContext) { + this.context = dlsContext; + } + + public boolean submit(DSLContext contextIn, long taskId, String tenantId, String callbackUrl, String taskName, String dockerName, String commands) { + return contextIn.insertInto(JUICE_TASK) + .set(JUICE_TASK.TASK_ID, taskId) + .set(JUICE_TASK.TENANT_ID, tenantId) + .set(JUICE_TASK.TASK_NAME, taskName) + .set(JUICE_TASK.TASK_STATUS, TaskResult.Result.NOT_START.getType()) + .set(JUICE_TASK.CALLBACK_URL, callbackUrl) + .set(JUICE_TASK.DOCKER_IMAGE, dockerName) + .set(JUICE_TASK.MESSAGE, "task is submit") + .set(JUICE_TASK.COMMANDS, commands) + .execute() > 0; + } + + public JuiceTask query(long taskId) { + return context.select() + .from(JUICE_TASK) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .fetchOneInto(JuiceTask.class); + } + + public JuiceTask query(String tenantId, long taskId) { + return context.select() + .from(JUICE_TASK) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .and(JUICE_TASK.TENANT_ID.eq(tenantId)) + .fetchOneInto(JuiceTask.class); + } + + public JuiceTask queryRunningTask(String tenantId, Long taskId) { + return context.select() + .from(JUICE_TASK) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .and(JUICE_TASK.TENANT_ID.eq(tenantId)) + .and(JUICE_TASK.TASK_STATUS.le(TaskResult.Result.RUNNING.getType())) + .fetchOneInto(JuiceTask.class); + } + + public List queryRunningTasks(String tenantId, List taskIds) { + return context.select() + .from(JUICE_TASK) + .where(JUICE_TASK.TASK_ID.in(taskIds)) + .and(JUICE_TASK.TENANT_ID.eq(tenantId)) + .and(JUICE_TASK.TASK_STATUS.le(TaskResult.Result.RUNNING.getType())) + .fetchInto(JuiceTask.class); + } + + public List queryTasks(String tenantId, List taskIds) { + return context.select() + .from(JUICE_TASK) + .where(JUICE_TASK.TASK_ID.in(taskIds)) + .and(JUICE_TASK.TENANT_ID.eq(tenantId)) + .fetchInto(JuiceTask.class); + } + + public boolean finish(long taskId, byte status, String message) { + return context.update(JUICE_TASK) + .set(JUICE_TASK.TASK_STATUS, status) + .set(JUICE_TASK.MESSAGE, message) + .set(JUICE_TASK.FINISH_AT, currentTimestamp()) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .and(JUICE_TASK.TASK_STATUS.ne(status)) + .execute() > 0; + } + + public boolean finishWithCallBack(long taskId, byte status, String message) { + return context.update(JUICE_TASK) + .set(JUICE_TASK.TASK_STATUS, status) + .set(JUICE_TASK.MESSAGE, message) + .set(JUICE_TASK.FINISH_AT, currentTimestamp()) + .set(JUICE_TASK.CALLBACK_AT, currentTimestamp()) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .and(JUICE_TASK.TASK_STATUS.ne(status)) + .execute() > 0; + } + + public boolean isExpired(long taskId, int expiredOfHours) { + if(expiredOfHours <= 0) { + expiredOfHours = EXPIRE_DURATION; + } + return context.select() + .from(JUICE_TASK) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .and(JUICE_TASK.TASK_STATUS.le(TaskResult.Result.RUNNING.getType())) + .and(JUICE_TASK.SUBMIT_AT.add(DayToSecond.valueOf(Duration.ofHours(expiredOfHours).toMillis())).le(currentTimestamp())) + .fetchOneInto(JuiceTask.class) != null; + } + + + public boolean update(long taskId, byte status, String message) { + return context.update(JUICE_TASK) + .set(JUICE_TASK.TASK_STATUS, status) + .set(JUICE_TASK.MESSAGE, message) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .and(JUICE_TASK.TASK_STATUS.ne(status)) + .execute() > 0; + } + + public boolean update(long taskId, String agent) { + return context.update(JUICE_TASK) + .set(JUICE_TASK.AGENT_ID, agent) + .where(JUICE_TASK.TASK_ID.eq(taskId)) + .execute() > 0; + } + + public boolean saveFrameworkId(String frameworkTag, String frameworkId) { + return context.insertInto(JUICE_FRAMEWORK) + .set(JUICE_FRAMEWORK.FRAMEWORK_TAG, frameworkTag) + .set(JUICE_FRAMEWORK.FRAMEWORK_ID, frameworkId) + .set(JUICE_FRAMEWORK.IS_ACTIVE, 1) + .onDuplicateKeyUpdate() + .set(JUICE_FRAMEWORK.FRAMEWORK_ID, frameworkId) + .set(JUICE_FRAMEWORK.IS_ACTIVE, 1) + .set(JUICE_FRAMEWORK.LAST_UPDATE_AT, currentTimestamp()) + .execute() > 0; + } + + public JuiceFramework queryFramework(String frameworkTag) { + return context.select() + .from(JUICE_FRAMEWORK) + .where(JUICE_FRAMEWORK.FRAMEWORK_TAG.eq(frameworkTag)) + .and(JUICE_FRAMEWORK.IS_ACTIVE.eq(1)) + .fetchOneInto(JuiceFramework.class); + } + + public boolean unActiveFramework(String frameworkTag) { + return context.update(JUICE_FRAMEWORK) + .set(JUICE_FRAMEWORK.IS_ACTIVE, 0) + .where(JUICE_FRAMEWORK.FRAMEWORK_TAG.eq(frameworkTag)) + .execute() > 0; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenUtil.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenUtil.java new file mode 100644 index 0000000..502ec27 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenUtil.java @@ -0,0 +1,74 @@ +package com.hujiang.juice.common.utils.generator; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 64位ID (42(毫秒)+10(机器随机数)+12(重复累加)) + * Created by xujia on 2017/03/21. + */ +public class IdGenUtil { + private final static long twepoch = 1288834974657L; + // 序列ID(12位) 1-12 + private final static long SEQUENCE_OFFSET = 0L; + private final static long SEQUENCE_LENGTH = 12L; + + // 机器ID(10位) 13-22 + private final static long MACHINE_OFFSET = SEQUENCE_OFFSET + SEQUENCE_LENGTH; + private final static long MACHINE_LENGTH = 10L; + + // 时间戳ID(42位) 23-64 + private final static long TIMESTAMP_OFFSET = MACHINE_OFFSET + MACHINE_LENGTH; + + private final static long machineId = Math.abs(UUID.randomUUID().hashCode() & 0X3FF); + private static AtomicLong sequence = new AtomicLong(0); + + @Deprecated + public static long genId(long bizId){ + return genId(); + } + + public static long genId(){ + + // System.out.println("sequence : " + Long.toBinaryString(sequence.get() & 0x3ff)); + // System.out.println("machineId : " + Long.toBinaryString(machineId << 10)); + // System.out.println("bizId : " + Long.toBinaryString(bizId << 15)); + // System.out.println("time : " + Long.toBinaryString((System.currentTimeMillis() - twepoch) << 22)); + + // ID偏移组合生成最终的ID，并返回ID + return ((System.currentTimeMillis() - twepoch) << TIMESTAMP_OFFSET) + | (machineId << MACHINE_OFFSET) + | (sequence.getAndIncrement() & 0XFFF); + } + + public static void main(String[] args){ + + Map> amap = Maps.newHashMap(); + int i = 0; + while(i < 500000) { + i++; + long id = genId(); + + if(amap.containsKey(id)){ + amap.get(id).add(sequence.get()); + + } else { + List longList = Lists.newArrayList(); + longList.add(sequence.get()); + amap.put(id, longList); + } + } + + amap.entrySet().stream().filter(v -> v.getValue().size() > 1).forEach(v -> {System.out.println("key : " + v.getKey() + ", value : " + v.getValue());}); + System.out.println("size : " + amap.entrySet().stream().filter(v -> v.getValue().size() > 1).count()); + + + // IdGenUtil gen = new IdGenUtil(4095); + // System.out.println(Long.toBinaryString(gen.nextId())); + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenerator.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenerator.java new file mode 100644 index 0000000..0e70afb --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/generator/IdGenerator.java @@ -0,0 +1,14 @@ +package com.hujiang.juice.common.utils.generator; + + +import static com.hujiang.juice.common.utils.generator.IdGenUtil.genId; + +/** + * Created by xujia on 16/11/11. + */ +public class IdGenerator { + public static long nextId() { + return genId(); + } +} + diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/OkHttpUtils.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/OkHttpUtils.java new file mode 100644 index 0000000..b075828 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/OkHttpUtils.java @@ -0,0 +1,222 @@ +package com.hujiang.juice.common.utils.rest; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; + +import javax.net.ssl.*; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Map; +import java.util.concurrent.TimeUnit; + + +/** + * Created by xujia on 16/6/8. + *

+ * config in startup.properties : + * ok.http.time.out=XX(default = 10) + * ok.http.retry.times=X(default = 3) + * if u don't wanner default value + */ + +@Slf4j +@Data +public class OkHttpUtils { + + private static OkHttpUtils instance; + + private OkHttpClient okHttpClient; + private OkHttpClient okHttpsClient; + + private long connectTimeOut; + private long readTimeOut; + private long writeTimeOut; + private int retrys; + + + private OkHttpClient.Builder getBuilder(long connectTimeOut, long readTimeOut, long writeTimeOut, int retrys) { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + return builder.connectTimeout(connectTimeOut, TimeUnit.SECONDS) + .readTimeout(readTimeOut, TimeUnit.SECONDS) + .writeTimeout(writeTimeOut, TimeUnit.SECONDS) + .addInterceptor( + new Interceptor() { + @Override + public Response intercept(Chain chain) throws IOException { + // try the request + Request request = chain.request(); + Response response = chain.proceed(request); + int tryCount = 0; + while (!response.isSuccessful() && + tryCount < retrys) { + response.close(); + response = chain.proceed(request); + tryCount++; + } + + return response; + } + } + ); + } + + private OkHttpUtils(long connectTimeOut, long readTimeOut, long writeTimeOut, int retrys) { + OkHttpClient.Builder builder = getBuilder(connectTimeOut, readTimeOut, writeTimeOut, retrys); + + this.connectTimeOut = connectTimeOut; + this.readTimeOut = readTimeOut; + this.writeTimeOut = writeTimeOut; + this.retrys = retrys; + + okHttpClient = builder.build(); + try { + ssl(builder); + okHttpsClient = builder.build(); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + log.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage()); + } + } + + public static OkHttpUtils getInstance(long connectTimeOut, long readTimeOut, long writeTimeOut, int retrys) { + if(null == instance) { + synchronized (OkHttpUtils.class) { + if(null == instance) { + instance = new OkHttpUtils(connectTimeOut, readTimeOut, writeTimeOut, retrys); + } + } + } + return instance; + } + + private boolean checkSSL(String urlStr) throws MalformedURLException { + URL url = new URL(urlStr); + return "https".equalsIgnoreCase(url.getProtocol()); + } + + private void ssl(OkHttpClient.Builder builder) throws NoSuchAlgorithmException, KeyManagementException { + SSLContext sc = SSLContext.getInstance("SSL"); + + sc.init(null, new TrustManager[]{ + trustManager + }, new SecureRandom()); + builder.sslSocketFactory(sc.getSocketFactory(), trustManager).hostnameVerifier(hostnameVerifier); + } + + private static final X509TrustManager trustManager = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + }; + + private static final HostnameVerifier hostnameVerifier = new HostnameVerifier() { + @Override + public boolean verify(String s, SSLSession sslSession) { + return true; + } + }; + + + public Response put(String url, RequestBody requestBody, Map headerMaps) throws IOException { + return doRequest("PUT", url, requestBody, headerMaps); + } + + public Response post(String url, RequestBody requestBody, Map headerMaps) throws IOException { + return doRequest("POST", url, requestBody, headerMaps); + } + + public Response delete(String url, RequestBody requestBody, Map headerMaps) throws IOException { + return doRequest("DELETE", url, requestBody, headerMaps); + } + + public Response get(String url, Map headerMaps) throws IOException { + return doRequest("GET", url, null, headerMaps); + } + + + public Response doRequest(String type, String url, RequestBody body, Map headerMaps) throws IOException { + return doOkRequest(type, url, body, headerMaps); + } + + + private OkHttpClient getClient(String url) throws MalformedURLException { + if(checkSSL(url)) { + return okHttpsClient; + } + return okHttpClient; + } + /** + * @param type request type: get, put, post or delete + * @param url + * @param body + * @param headerMaps + * @return + * @throws IOException + */ + public Response doOkRequest(String type, String url, RequestBody body, Map headerMaps) throws IOException { + Response response = null; + try { + + Request.Builder builder = new Request.Builder().url(url); + + if (null != headerMaps && headerMaps.size() > 0) + builder.headers(Headers.of(headerMaps)); + + switch (type) { + case "PUT": + case "POST": + case "DELETE": + case "PATCH": + if(null == body) { + body = RequestBody.create(null, new byte[0]); + } + + builder.method(type, body); + break; + case "GET": + case "HEAD": + builder.method(type, null); + break; + } + Request request = builder.build(); + + OkHttpClient client = getClient(url); + response = client.newCall(request).execute(); + + if (!response.isSuccessful()) { + + int code = response.code(); + StringBuilder message = new StringBuilder(); + message.append(response.message()); + response.close(); + + log.warn("Unexpected code :" + code + ", cause : " + message.toString()); + throw new IOException(message.toString()); + } + return response; + } catch (IOException e) { + if (null != response) { + response.close(); + } + throw e; + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/ParameterTypeReference.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/ParameterTypeReference.java new file mode 100644 index 0000000..e856605 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/ParameterTypeReference.java @@ -0,0 +1,65 @@ +package com.hujiang.juice.common.utils.rest; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; + +/** + * Created by xujia on 2016/6/14. + */ +public abstract class ParameterTypeReference { + private final Type type; + + public static void notNull(Object object, String message) { + if(object == null) { + throw new IllegalArgumentException(message); + } + } + + public static void isInstanceOf(Class clazz, Object obj) { + notNull(clazz, "Type to check against must not be null"); + if(!clazz.isInstance(obj)) { + throw new IllegalArgumentException("Object of class [" + (obj != null?obj.getClass().getName():"null") + "] must be an instance of " + clazz); + } + } + + public static void isTrue(boolean expression) { + if(!expression) { + throw new IllegalArgumentException("[Assertion failed] - this expression must be true"); + } + } + + protected ParameterTypeReference() { + Class parameterTypeReferenceSubclass = findParameterTypeReferenceSubclass(this.getClass()); + Type type = parameterTypeReferenceSubclass.getGenericSuperclass(); + isInstanceOf(ParameterizedType.class, type); + ParameterizedType parameterizedType = (ParameterizedType)type; + isTrue(parameterizedType.getActualTypeArguments().length == 1); + this.type = parameterizedType.getActualTypeArguments()[0]; + } + + public Type getType() { + return this.type; + } + + public boolean equals(Object obj) { + return this == obj || obj instanceof ParameterTypeReference && this.type.equals(((ParameterTypeReference)obj).type); + } + + public int hashCode() { + return this.type.hashCode(); + } + + public String toString() { + return "ParameterTypeReference<" + this.type + ">"; + } + + private static Class findParameterTypeReferenceSubclass(Class child) { + Class parent = child.getSuperclass(); + if(Object.class == parent) { + throw new IllegalStateException("Expected ParameterTypeReference superclass"); + } else { + return ParameterTypeReference.class == parent?child:findParameterTypeReferenceSubclass(parent); + } + } +} + diff --git a/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/Restty.java b/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/Restty.java new file mode 100644 index 0000000..968dad0 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/utils/rest/Restty.java @@ -0,0 +1,367 @@ +package com.hujiang.juice.common.utils.rest; + +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import okhttp3.MediaType; +import okhttp3.RequestBody; +import okhttp3.Response; +import okio.ByteString; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.hujiang.juice.common.utils.rest.Restty.DefaultHttpClientSetting.*; + + +/** + * Created by xujia on 16/6/7. + */ +@Slf4j +@Data +public class Restty { + + private Gson gson = new Gson(); + + private OkHttpUtils okHttpUtils; + + private String url = null; + private RequestBody requestBody = null; + private Map headers = null; + private MediaType mediaType = null; + + private long connectTimeOut; + private long readTimeOut; + private long writeTimeOut; + private int retrys; + private boolean emptyParameter = false; + + public static String protoBody() { + return BodyType.PROTOBUF.getType(); + } + + public static String textBody() { + return BodyType.TEXT.getType(); + } + + public static String streamBody() { + return BodyType.STREAMS.getType(); + } + + public static String jsonBody() { + return BodyType.JSON.getType(); + } + + + private Restty(String url, long connectTimeOut, long readTimeOut, long writeTimeOut, int retrys) { + this.url = url; + this.connectTimeOut = connectTimeOut; + this.readTimeOut = readTimeOut; + this.writeTimeOut = writeTimeOut; + this.retrys = retrys; + okHttpUtils = OkHttpUtils.getInstance(connectTimeOut, readTimeOut, writeTimeOut, retrys); + } + + public static Restty create(String url) { + return new Restty(url, CONNECT_TIME_OUT, READ_TIME_OUT, WRITE_TIME_OUT, OK_HTTP_RETRY_TIMES); + } + + public static Restty create(String url, Map mapParams) { + return new Restty(url, CONNECT_TIME_OUT, READ_TIME_OUT, WRITE_TIME_OUT, OK_HTTP_RETRY_TIMES).addAllParameters(mapParams); + } + + public static Restty create(String url, long connectTimeOut, long readTimeOut, long writeTimeOut, int retrys) { + return new Restty(url, connectTimeOut, readTimeOut, writeTimeOut, retrys); + } + + public static Restty create(String url, long connectTimeOut, long readTimeOut, long writeTimeOut, int retrys, Map mapParams) { + return new Restty(url, connectTimeOut, readTimeOut, writeTimeOut, retrys).addAllParameters(mapParams); + } + + public Restty addAllParameters(Map mapParams) { + StringBuilder sb = new StringBuilder(); + if (null != mapParams && mapParams.size() > 0) { + for (Map.Entry entry : mapParams.entrySet()) { + sb.append(generateParameterString(entry.getKey(), entry.getValue())); + } + } + this.url += sb.toString(); + return this; + } + + public Restty addParameter(String key, String value) { + this.url += generateParameterString(key, value); + return this; + } + + public Restty addParameters(String key, List values) { + if (null == values || values.size() == 0) { + return this; + } + String valueString = values.stream().map(Object::toString).collect(Collectors.joining(",")); + + return addParameter(key, valueString); + } + + private String generateParameterString(String key, String value) { + StringBuilder sb = new StringBuilder(); + if (!emptyParameter) { + synchronized (this) { + if(!emptyParameter) { + sb.append("?"); + emptyParameter = true; + } else { + sb.append("&"); + } + } + } else { + sb.append("&"); + } + sb.append(key).append("=").append(value); + return sb.toString(); + } + + + public Restty addMediaType(String mediaType) { + this.mediaType = MediaType.parse(mediaType); + return this; + } + + public Restty addKeepAlive() { + addHeader("Connection", "keep-alive"); + return this; + } + + public Restty addAccept(String value) { + addHeader("Accept", value); + return this; + } + + public Restty addHeader(String name, String value) { + if (null == headers) { + headers = Maps.newHashMap(); + } + headers.put(name, value); + return this; + } + + public Restty addHeaders(Map headers) { + this.headers = headers; + return this; + } + + public Restty requestBody(Object body) { + requestBody = RequestBody.create(mediaType, gson.toJson(body)); + return this; + } + + public Restty requestBody(String body) { + requestBody = RequestBody.create(mediaType, body); + return this; + } + + public Restty requestBody(File body) { + requestBody = RequestBody.create(mediaType, body); + return this; + } + + public Restty requestBody(ByteString body) { + requestBody = RequestBody.create(mediaType, body); + return this; + } + + public Restty requestBody(byte[] body) { + requestBody = RequestBody.create(mediaType, body); + return this; + } + + public Restty requestBody(byte[] body, int offset, int byteCount) { + requestBody = RequestBody.create(mediaType, body, offset, byteCount); + return this; + } + + // post + public T post(ParameterTypeReference parameterTypeReference) throws IOException { + try (Response response = post()) { + return null == response.body() ? null : gson.fromJson(response.body().charStream(), parameterTypeReference.getType()); + } + } + + // this function must close response manually + public Response post() throws IOException { + return okHttpUtils.post(url, requestBody, headers); + } + + public void postNoResponse() throws IOException { + Response response = null; + try { + response = okHttpUtils.post(url, requestBody, headers); + } finally { + if(null != response) { + response.close(); + } + } + } + + public byte[] postBytes() throws IOException { + Response response = okHttpUtils.post(url, requestBody, headers); + return null == response ? null : response.body().bytes(); + } + + public String postString() throws IOException { + Response response = okHttpUtils.post(url, requestBody, headers); + return null == response ? null : response.body().string(); + } + + // this function must close response manually + public InputStream postStream() throws IOException { + Response response = okHttpUtils.post(url, requestBody, headers); + return null == response ? null : response.body().byteStream(); + } + + // get + public T get(ParameterTypeReference parameterTypeReference) throws IOException { + try (Response response = get()) { + return null == response.body() ? null : gson.fromJson(response.body().charStream(), parameterTypeReference.getType()); + } + } + + public Response get() throws IOException { + return okHttpUtils.get(url, headers); + } + + public void getNoResponse() throws IOException { + Response response = null; + try { + response = okHttpUtils.get(url, headers); + } finally { + if(null != response) { + response.close(); + } + } + } + + public byte[] getBytes() throws IOException { + Response response = okHttpUtils.get(url, headers); + return null == response ? null : response.body().bytes(); + } + + public String getString() throws IOException { + Response response = okHttpUtils.get(url, headers); + return null == response ? null : response.body().string(); + } + + // this function must close response manually + public InputStream getStream() throws IOException { + Response response = okHttpUtils.get(url, headers); + return null == response ? null : response.body().byteStream(); + } + + // put + public T put(ParameterTypeReference parameterTypeReference) throws IOException { + try (Response response = put()) { + return null == response.body() ? null : gson.fromJson(response.body().charStream(), parameterTypeReference.getType()); + } + } + + // this function must close response manually + public Response put() throws IOException { + return okHttpUtils.put(url, requestBody, headers); + } + + public void putNoResponse() throws IOException { + Response response = null; + try { + response = okHttpUtils.put(url, requestBody, headers); + } finally { + if(null != response) { + response.close(); + } + } + } + + public byte[] putBytes() throws IOException { + Response response = okHttpUtils.put(url, requestBody, headers); + return null == response ? null : response.body().bytes(); + } + + public String putString() throws IOException { + Response response = okHttpUtils.put(url, requestBody, headers); + return null == response ? null : response.body().string(); + } + + // this function must close response manually + public InputStream putStream() throws IOException { + Response response = okHttpUtils.put(url, requestBody, headers); + return null == response ? null : response.body().byteStream(); + } + + // delete + public T delete(ParameterTypeReference parameterTypeReference) throws IOException { + try (Response response = delete()) { + return null == response.body() ? null : gson.fromJson(response.body().charStream(), parameterTypeReference.getType()); + } + } + + // this function must close response manually + public Response delete() throws IOException { + return okHttpUtils.delete(url, requestBody, headers); + } + + public void deleteNoResponse() throws IOException { + Response response = null; + try { + response = okHttpUtils.delete(url, requestBody, headers); + } finally { + if(null != response) { + response.close(); + } + } + } + + public byte[] deleteBytes() throws IOException { + Response response = okHttpUtils.delete(url, requestBody, headers); + return null == response ? null : response.body().bytes(); + } + + public String deleteString() throws IOException { + Response response = okHttpUtils.delete(url, requestBody, headers); + return null == response ? null : response.body().string(); + } + + // this function must close response manually + public InputStream deleteStream() throws IOException { + Response response = okHttpUtils.delete(url, requestBody, headers); + return null == response ? null : response.body().byteStream(); + } + + enum BodyType { + TEXT("text/plain"), + STREAMS("octet-stream"), + JSON("application/json"), + PROTOBUF("application/x-protobuf"); + + private String type; + + BodyType(String type) { + this.type = type; + } + + public String getType() { + return type; + } + } + + class DefaultHttpClientSetting { + + public static final long CONNECT_TIME_OUT = 20; + public static final long READ_TIME_OUT = 20; + public static final long WRITE_TIME_OUT = 20; + public static final int OK_HTTP_RETRY_TIMES = 1; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/vo/Result.java b/juice-common/src/main/java/com/hujiang/juice/common/vo/Result.java new file mode 100644 index 0000000..83fb776 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/vo/Result.java @@ -0,0 +1,46 @@ +package com.hujiang.juice.common.vo; + +import java.io.Serializable; + +/** + * Created by xujia on 2016/6/14. + */ +public class Result implements Serializable { + private static final long serialVersionUID = 2120467584344923858L; + private Integer status = Integer.valueOf(0); + private String message = null; + private T data = null; + + public Result() { + } + + public Result(Integer status, String message, T data) { + this.status = status; + this.message = message; + this.data = data; + } + + public Integer getStatus() { + return this.status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getMessage() { + return this.message; + } + + public void setMessage(String message) { + this.message = message; + } + + public T getData() { + return this.data; + } + + public void setData(T data) { + this.data = data; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/vo/ResultBuilder.java b/juice-common/src/main/java/com/hujiang/juice/common/vo/ResultBuilder.java new file mode 100644 index 0000000..9780430 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/vo/ResultBuilder.java @@ -0,0 +1,34 @@ +package com.hujiang.juice.common.vo; + + +/** + * Created by xujia on 2016/6/14. + */ +public class ResultBuilder { + private Integer status = Integer.valueOf(0); + private String message; + private T data; + + public ResultBuilder() { + } + + public ResultBuilder status(Integer status) { + this.status = status; + return this; + } + + public ResultBuilder message(String message) { + this.message = message; + return this; + } + + public ResultBuilder data(T data) { + this.data = data; + return this; + } + + public Result build() { + return new Result(this.status, this.message, this.data); + } +} + diff --git a/juice-common/src/main/java/com/hujiang/juice/common/vo/SubmitTask.java b/juice-common/src/main/java/com/hujiang/juice/common/vo/SubmitTask.java new file mode 100644 index 0000000..229daf3 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/vo/SubmitTask.java @@ -0,0 +1,66 @@ +package com.hujiang.juice.common.vo; + +import com.hujiang.juice.common.model.*; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + + +/** + * Created by xujia on 16/11/8. + */ + +@Data +@Slf4j +public class SubmitTask { + + private Resources resources; + private Container container; + private Command.Environment env; + private List args; + private String commands; + + private Constraints constraints; + private String taskName; + private RunModel runMode; + private String callbackUrl; + private Long taskId; + + public SubmitTask() { + + } + + public SubmitTask(String taskName, String callbackUrl, Command.Environment env, List args, Constraints constraints, Resources resources, Container container) { + this.taskName = taskName; + this.callbackUrl = callbackUrl; + this.env = env; + this.args = args; + this.resources = resources; + this.container = container; + this.constraints = constraints; + } + + public SubmitTask(String taskName, String callbackUrl, Command.Environment env, List args, Resources resources, Constraints constraints, String commands) { + this.taskName = taskName; + this.callbackUrl = callbackUrl; + this.env = env; + this.args = args; + this.args = args; + this.resources = resources; + this.commands = commands; + this.constraints = constraints; + } + + public Task toTask() { + if (runMode == RunModel.COMMAND) { + return new Task(resources, container, new Command(commands, env, null), constraints, taskName, taskId); + } else { + return new Task(resources, container, new Command(null, env, args), constraints, taskName, taskId); + } + } + + public enum RunModel { + COMMAND, CONTAINER + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskKill.java b/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskKill.java new file mode 100644 index 0000000..4f569b6 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskKill.java @@ -0,0 +1,20 @@ +package com.hujiang.juice.common.vo; + +import lombok.Data; + +/** + * Created by xujia on 17/3/21. + */ + +@Data +public class TaskKill { + private boolean submitTask; + private byte currentStatus; + private String resultMessage; + + public TaskKill(boolean submitTask, byte currentStatus, String resultMessage) { + this.submitTask = submitTask; + this.currentStatus = currentStatus; + this.resultMessage = resultMessage; + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskReconcile.java b/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskReconcile.java new file mode 100644 index 0000000..254ffd2 --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskReconcile.java @@ -0,0 +1,37 @@ +package com.hujiang.juice.common.vo; + +import lombok.Data; + +import java.util.List; + + +/** + * Created by xujia on 17/2/7. + */ + +@Data +public class TaskReconcile { + + private int request; + private int reconcile; + private List reconciles; + + public TaskReconcile(int request, int reconcile, List reconciles) { + this.request = request; + this.reconcile = reconcile; + this.reconciles = reconciles; + } + + @Data + public static class Reconcile { + private long taskId; + private boolean reconciled; + private String message; + + public Reconcile(long taskId, boolean reconciled, String message){ + this.taskId = taskId; + this.reconciled = reconciled; + this.message = message; + } + } +} diff --git a/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskResult.java b/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskResult.java new file mode 100644 index 0000000..db6101d --- /dev/null +++ b/juice-common/src/main/java/com/hujiang/juice/common/vo/TaskResult.java @@ -0,0 +1,58 @@ +package com.hujiang.juice.common.vo; + +import lombok.Data; + +import java.util.Arrays; +import java.util.Optional; + +/** + * Created by xujia on 16/11/30. + */ + +@Data +public class TaskResult { + private long taskId; + private Result result; + private String message; + + public TaskResult(long taskId, Result result, String message) { + this.taskId = taskId; + this.result = result; + this.message = message; + } + + public enum Result { + NOT_START(Byte.valueOf("-1")), + STAGING(Byte.valueOf("0")), + RUNNING(Byte.valueOf("1")), + FINISHED(Byte.valueOf("2")), + FAILED(Byte.valueOf("3")), + LOST(Byte.valueOf("4")), + ERROR(Byte.valueOf("5")), + KILLED(Byte.valueOf("6")), + UNREACHABLE(Byte.valueOf("7")), + DROPPED(Byte.valueOf("8")), + GONE(Byte.valueOf("9")), + GONE_BY_OPERATOR(Byte.valueOf("10")), + UNKNOWN(Byte.valueOf("11")), + EXPIRED(Byte.valueOf("12")); + + private byte type; + + public byte getType() { + return type; + } + + Result(byte type) { + this.type = type; + } + + public static String getName(byte b) { + Optional result = Arrays.stream(Result.values()).filter(v -> v.getType() == b).findFirst(); + if(result.isPresent()) { + return result.get().name(); + } + return null; + } + } +} diff --git a/juice-jooq/.gitignore b/juice-jooq/.gitignore new file mode 100644 index 0000000..1440ef7 --- /dev/null +++ b/juice-jooq/.gitignore @@ -0,0 +1,26 @@ +# Created by .ignore support plugin (hsz.mobi) +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### Example user template template +### Example user template + +# IntelliJ project files +.idea/ +.settings/ +../juice-rest/target/ +.project +.classpath +*.iml +out +gen diff --git a/juice-jooq/pom.xml b/juice-jooq/pom.xml new file mode 100644 index 0000000..4d117cf --- /dev/null +++ b/juice-jooq/pom.xml @@ -0,0 +1,77 @@ + + + + juice + com.hujiang + 1.0-OPEN + + 4.0.0 + + juice-jooq + + + + org.jooq + jooq + + + org.jooq + jooq-meta + + + org.jooq + jooq-codegen + + + mysql + mysql-connector-java + + + + + + + org.jooq + jooq-codegen-maven + ${jooq.version} + + + generate-mysql + generate-sources + + generate + + + + com.mysql.jdbc.Driver + jdbc:mysql://your ip:your port/juice + your username + your password + + + + org.jooq.util.mysql.MySQLDatabase + juice_task|juice_framework + juice + + true + + + true + + + + + com.hujiang.jooq.juice + target/generated-sources/jooq-mysql + + + + + + + + + \ No newline at end of file diff --git a/juice-rest/pom.xml b/juice-rest/pom.xml new file mode 100644 index 0000000..cef50ff --- /dev/null +++ b/juice-rest/pom.xml @@ -0,0 +1,96 @@ + + + + juice + com.hujiang + 1.0-OPEN + + 4.0.0 + + juice-rest + + 1.5.0.RELEASE + UTF-8 + 1.8 + com.hujiang.juice.rest.Startup + + + + + com.hujiang + juice-common + + + org.springframework.boot + spring-boot-starter-web + ${spring-boot.version} + + + org.springframework.boot + spring-boot-starter-tomcat + + + + + org.springframework.boot + spring-boot-starter-jetty + ${spring-boot.version} + + + org.springframework.boot + spring-boot-starter-data-redis + ${spring-boot.version} + + + org.springframework.boot + spring-boot-starter-jdbc + ${spring-boot.version} + + + org.springframework.boot + spring-boot-starter-jooq + ${spring-boot.version} + + + org.springframework.boot + spring-boot-starter-test + ${spring-boot.version} + + + com.alibaba + druid + + + + + juice-rest + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + org.springframework.boot + spring-boot-maven-plugin + + + + repackage + + + + + ${main-class} + + + + + \ No newline at end of file diff --git a/juice-rest/scripts/Dockerfile b/juice-rest/scripts/Dockerfile new file mode 100644 index 0000000..2b8a904 --- /dev/null +++ b/juice-rest/scripts/Dockerfile @@ -0,0 +1,17 @@ +FROM java:8-jre-alpine + +#add timezone and default it to Shanghai +RUN apk --update add --no-cache tzdata +ENV TZ=Asia/Shanghai + +RUN mkdir -p /app/log +COPY juice-rest.jar /app/juice-rest.jar +COPY entrypoint.sh /app/entrypoint.sh + +RUN chmod +x /app/entrypoint.sh + +VOLUME ["/app/log"] +WORKDIR /app/ + +ENTRYPOINT ["./entrypoint.sh"] +CMD [] \ No newline at end of file diff --git a/juice-rest/scripts/entrypoint.sh b/juice-rest/scripts/entrypoint.sh new file mode 100644 index 0000000..49ef040 --- /dev/null +++ b/juice-rest/scripts/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd /app + +echo "environment=${environment}" + +java -Dspring.profiles.active=${environment} -jar juice-rest.jar \ No newline at end of file diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/Startup.java b/juice-rest/src/main/java/com/hujiang/juice/rest/Startup.java new file mode 100644 index 0000000..9627356 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/Startup.java @@ -0,0 +1,20 @@ +package com.hujiang.juice.rest; + +import lombok.extern.slf4j.Slf4j; +import com.hujiang.juice.rest.config.AppConfig; +import org.springframework.boot.builder.SpringApplicationBuilder; + +/** + * Created by xujia on 16/12/5. + */ + +@Slf4j +public class Startup { + + public static void main(String[] args) { + new SpringApplicationBuilder() + .sources(AppConfig.class) + .build() + .run(args); + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/config/AppConfig.java b/juice-rest/src/main/java/com/hujiang/juice/rest/config/AppConfig.java new file mode 100644 index 0000000..5ef9698 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/config/AppConfig.java @@ -0,0 +1,76 @@ +package com.hujiang.juice.rest.config; + +import com.hujiang.juice.common.utils.cache.CacheUtils; +import com.hujiang.juice.common.utils.cache.RedisCacheUtils; +import com.hujiang.juice.common.utils.cache.RedisUtil; +import com.hujiang.juice.common.utils.db.DaoUtils; +import com.hujiang.juice.common.utils.db.JuiceDao; +import com.hujiang.juice.rest.utils.SubscriberUtils; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * Created by xujia on 17/2/12. + */ + + +@Slf4j +@Configuration +@ComponentScan("com.hujiang") +@SpringBootApplication +@EnableAutoConfiguration() +@EnableScheduling +@Import({CacheConfig.class}) +public class AppConfig { + + @Value("${spring.profiles.active}") + private String profileActive; + + @Value("${juice.task.queue:juice.task.queue.}") + private String taskQueue; + + @Value("${juice.task.result.queue:juice.task.result.queue.}") + private String taskResultQueue; + + @Value("${juice.management.queue:juice.management.queue.}") + private String managementQueue; + + @Value("${juice.task.expired.of.seconds:86400}") + private int expiredSeconds; + + @Autowired + private DSLContext dslContext; + + @Autowired + private RedisUtil redisUtil; + + @Bean + public CacheUtils getCacheUtils() { + return new RedisCacheUtils(redisUtil); + } + + @Bean + public SubscriberUtils getSubscriberUtils() { + return new SubscriberUtils(); + } + + @Bean + public DaoUtils getDaoService() { + return new DaoUtils(new JuiceDao(dslContext)); + } + + @Bean + public CachesBizConfig getCachesBizConfig() { + log.info("profileActive : " + profileActive); + return new CachesBizConfig(taskQueue + profileActive, null, taskResultQueue + profileActive, managementQueue + profileActive, expiredSeconds); + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/config/CacheConfig.java b/juice-rest/src/main/java/com/hujiang/juice/rest/config/CacheConfig.java new file mode 100644 index 0000000..77004bc --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/config/CacheConfig.java @@ -0,0 +1,26 @@ +package com.hujiang.juice.rest.config; + +import com.hujiang.juice.common.utils.cache.RedisUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; + +/** + * Created by xujia on 17/2/12. + */ +public class CacheConfig { + + @Value("${spring.redis.host}") + private String redisIp; + + @Value("${spring.redis.port}") + private int port; + + @Value("${spring.redis.password}") + private String password; + + @Bean + public RedisUtil getRedisUtil() { + return new RedisUtil(redisIp, port, password); + } + +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/config/CachesBizConfig.java b/juice-rest/src/main/java/com/hujiang/juice/rest/config/CachesBizConfig.java new file mode 100644 index 0000000..46a1e67 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/config/CachesBizConfig.java @@ -0,0 +1,33 @@ +package com.hujiang.juice.rest.config; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +/** + * Created by xujia on 17/2/12. + */ + +@Data +@Slf4j +public class CachesBizConfig { + private String taskQueue; + private String taskRetryQueue; + private String resultQueue; + private String managementQueue; + private int expiredSeconds; + + + + public CachesBizConfig(String taskQueue, String taskRetryQueue, String resultQueue, String managementQueue, int expiredSeconds) { + this.taskQueue = taskQueue; + this.taskRetryQueue = taskRetryQueue; + this.resultQueue = resultQueue; + this.managementQueue = managementQueue; + this.expiredSeconds = expiredSeconds; + + log.info("taskQueue : " + taskQueue); + log.info("resultQueue : " + resultQueue); + log.info("managementQueue : " + managementQueue); + log.info("expiredOfSeconds : " + expiredSeconds); + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/package-info.java b/juice-rest/src/main/java/com/hujiang/juice/rest/package-info.java new file mode 100644 index 0000000..8efda96 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/package-info.java @@ -0,0 +1,4 @@ + /** + * Created by xujia on 16/12/5. + */ +package com.hujiang.juice.rest; \ No newline at end of file diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/service/RestService.java b/juice-rest/src/main/java/com/hujiang/juice/rest/service/RestService.java new file mode 100644 index 0000000..9f7bd53 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/service/RestService.java @@ -0,0 +1,166 @@ +package com.hujiang.juice.rest.service; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.common.error.CommonStatusCode; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.exception.RestException; +import com.hujiang.juice.common.model.TaskManagement; +import com.hujiang.juice.common.utils.cache.CacheUtils; +import com.hujiang.juice.common.utils.generator.IdGenerator; +import com.hujiang.juice.common.vo.SubmitTask; +import com.hujiang.juice.common.vo.TaskKill; +import com.hujiang.juice.common.vo.TaskResult; + +import com.hujiang.juice.common.vo.TaskReconcile; +import com.hujiang.juice.common.utils.db.DaoUtils; +import com.hujiang.juice.rest.config.CachesBizConfig; +import com.hujiang.juice.rest.utils.TaskUtils; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +import static com.hujiang.juice.common.config.COMMON.KILL; +import static com.hujiang.juice.common.config.COMMON.RECONCILE; + +/** + * Created by xujia on 16/12/5. + */ + +@Slf4j +@Data +@Service +public class RestService { + + @Autowired + private DaoUtils daoUtils; + + @Autowired + private CacheUtils cacheUtils; + + @Autowired + private CachesBizConfig cachesBizConfig; + + private Gson gson = new Gson(); + + private static final String POINT = "."; + + public long submits(SubmitTask object, String tenantId) { + + // set run mode + object.setRunMode(TaskUtils.checkRunMode(object)); + + // generate taskId + object.setTaskId(IdGenerator.nextId()); + log.info("TaskId --> " + object.getTaskId() + ", TaskName : " + object.getTaskName()); + + // submit(record in db and submit in redis) + daoUtils.getContext().transaction( + configuration -> { + String commands = ""; + String dockerImage = ""; + if (object.getRunMode() == SubmitTask.RunModel.CONTAINER) { + dockerImage = object.getContainer().getDocker().getImage(); + } else { + commands = object.getCommands(); + } + boolean isInsert = daoUtils.submit(configuration, object.getTaskId(), tenantId, object.getCallbackUrl(), object.getTaskName(), dockerImage, commands); + if (isInsert) { + cacheUtils.pushToQueue(cachesBizConfig.getTaskQueue(), gson.toJson(object.toTask())); + } + } + ); + + return object.getTaskId(); + } + + public TaskKill kills(String tenantId, long taskId) { + JuiceTask task = daoUtils.queryTask(tenantId, taskId); + + if (null == task) { + throw new RestException(CommonStatusCode.QUERY_RECORD_EMPTY.getStatus(), "task not exist to kill!"); + } + + if (task.getTaskStatus() > TaskResult.Result.RUNNING.getType()) { + return new TaskKill(false, task.getTaskStatus(), task.getMessage()); + } + + TaskManagement taskManagement = new TaskManagement(Lists.newCopyOnWriteArrayList(), KILL); + TaskManagement.TaskAgentRel taskAgentRel = new TaskManagement.TaskAgentRel(task.getTaskId(), task.getTaskName(), task.getAgentId()); + taskManagement.getTaskAgentRels().add(taskAgentRel); +// +// String key = cachesBizConfig.getKillKeyPrefix() + POINT + taskId; +// cacheUtils.setExpired(key, "1", cachesBizConfig.getExpiredSeconds()); + cacheUtils.pushToQueue(cachesBizConfig.getManagementQueue(), gson.toJson(taskManagement)); + return new TaskKill(true, task.getTaskStatus(), "juice accept kill task command"); + + } + + public List querys(String tenantId, List taskId) { + return daoUtils.queryTasks(tenantId, taskId); + } + + public TaskReconcile reconciles(String tenantId, List taskIds) { + List tasks = daoUtils.queryTasks(tenantId, taskIds); + Map reconcileMap = getTaskReconcile(taskIds); + TaskManagement taskManagement = new TaskManagement(Lists.newCopyOnWriteArrayList(), RECONCILE); + tasks.stream().parallel().forEach(t -> { + String value = t.getAgentId(); + + boolean isReconciled = false; + String message = ""; + TaskReconcile.Reconcile reconcile = reconcileMap.get(t.getTaskId()); + if (null == reconcile) { + String error = "taskId not matched with database record, taskId: " + t.getTaskId(); + log.warn(error); + throw new RestException(ErrorCode.OBJECT_NOT_EQUAL_ERROR.getCode(), error); + } else if (!t.getTaskStatus().equals(TaskResult.Result.RUNNING.getType())) { + message = "not reconcile due to terminal task status : " + TaskResult.Result.getName(t.getTaskStatus()); + } else if (StringUtils.isBlank(value)) { + reconcile.setReconciled(false); + daoUtils.finishTask(t.getTaskId(), TaskResult.Result.EXPIRED.getType(), "task expired"); + message = "not reconcile due to terminal task status : " + TaskResult.Result.EXPIRED.name(); + } else { + TaskManagement.TaskAgentRel taskAgentRel = new TaskManagement.TaskAgentRel(t.getTaskId(), t.getTaskName(), value); + taskManagement.getTaskAgentRels().add(taskAgentRel); + isReconciled = true; + message = "reconcile task"; + } + + reconcile.setTaskId(t.getTaskId()); + reconcile.setReconciled(isReconciled); + reconcile.setMessage(message); + + }); + int reconcileCount = taskManagement.getTaskAgentRels().size(); + if (reconcileCount > 0) { + cacheUtils.pushToQueue(cachesBizConfig.getManagementQueue(), gson.toJson(taskManagement)); + } + return new TaskReconcile(taskIds.size(), reconcileCount, mapsToLists(reconcileMap)); + } + + private Map getTaskReconcile(List tasks) { + Map reconcileMap = Maps.newConcurrentMap(); + tasks.stream().parallel().forEach(v -> { + reconcileMap.put(v, new TaskReconcile.Reconcile(v, false, "invalid taskId")); + }); + return reconcileMap; + } + + private List mapsToLists(Map map) { + final List reconciles = Lists.newCopyOnWriteArrayList(); + map.entrySet().parallelStream().forEach( + v -> { + reconciles.add(v.getValue()); + } + ); + return reconciles; + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/service/SchedulerService.java b/juice-rest/src/main/java/com/hujiang/juice/rest/service/SchedulerService.java new file mode 100644 index 0000000..53253dd --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/service/SchedulerService.java @@ -0,0 +1,22 @@ +package com.hujiang.juice.rest.service; + +import com.hujiang.juice.rest.utils.SubscriberUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * Created by xujia on 17/2/12. + */ + +@Service +public class SchedulerService { + + @Autowired + private SubscriberUtils subscriberUtils; + + @Scheduled(fixedDelay = 1000) + public void handler() { + subscriberUtils.handleResult(); + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/utils/SubscriberUtils.java b/juice-rest/src/main/java/com/hujiang/juice/rest/utils/SubscriberUtils.java new file mode 100644 index 0000000..4a1df13 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/utils/SubscriberUtils.java @@ -0,0 +1,131 @@ +package com.hujiang.juice.rest.utils; + +import com.google.gson.Gson; +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.exception.CacheException; +import com.hujiang.juice.common.exception.DataBaseException; +import com.hujiang.juice.common.exception.RestException; +import com.hujiang.juice.common.utils.cache.CacheUtils; +import com.hujiang.juice.common.utils.db.DaoUtils; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.common.vo.TaskResult; +import com.hujiang.juice.rest.config.CachesBizConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + + +/** + * Created by xujia on 16/12/8. + */ +@Slf4j +public class SubscriberUtils { + + @Autowired + private DaoUtils daoUtils; + + @Autowired + private CacheUtils cacheUtils; + + @Autowired + private CachesBizConfig cachesBizConfig; + + private ExecutorService fixedThreadPool; + + private Gson gson = new Gson(); + + public SubscriberUtils() { + fixedThreadPool = Executors.newFixedThreadPool(10); + log.info("init SubscriberUtils success!"); + } + + public void handleResult() { + while (true) { + TaskResult taskResult = getTaskResult(); + if (null == taskResult) { + break; + } + log.debug("taskResult : " + taskResult.toString()); + fixedThreadPool.execute(() -> { + String callbackUrl = ""; + try { + JuiceTask task = daoUtils.queryTask(taskResult.getTaskId()); + if(null != task) { + callbackUrl = task.getCallbackUrl(); + if(StringUtils.isNotBlank(callbackUrl)) { + boolean isUpdate; + if (taskResult.getResult().getType() > TaskResult.Result.RUNNING.getType()) { + isUpdate = daoUtils.finishTaskWithCallBack(taskResult.getTaskId(), taskResult.getResult().getType(), taskResult.getMessage()); + } else { + isUpdate = daoUtils.updateTask(taskResult.getTaskId(), taskResult.getResult().getType(), taskResult.getMessage()); + } + if(isUpdate) { + log.debug("url --> " + callbackUrl); + log.debug("taskResult --> " + gson.toJson(taskResult)); + Restty.create(task.getCallbackUrl()) + .addHeader("X-Tenant-ID", task.getTenantId()) + .addMediaType(Restty.jsonBody()) + .requestBody(taskResult) + .postNoResponse(); + } + } else { + log.debug("not call back due to url is null, taskId : " + task.getTaskId()); + if(taskResult.getResult().getType() > TaskResult.Result.RUNNING.getType()) { + daoUtils.finishTask(taskResult.getTaskId(), taskResult.getResult().getType(), taskResult.getMessage()); + } else { + daoUtils.updateTask(taskResult.getTaskId(), taskResult.getResult().getType(), taskResult.getMessage()); + } + } + } else { + log.warn("scheduler service --> task : " + taskResult.getTaskId() + " not exist, can't update task status"); + } + } catch (Exception ex) { + if (ex instanceof DataBaseException) { + log.error("db operating error, cause : " + ex); + log.error("notice & check, taskResult : " + taskResult.toString()); + } else if (ex instanceof IOException) { + log.error("call back failed, url : " + callbackUrl); + throw new RestException(ErrorCode.HTTP_REQUEST_ERROR.getCode(), ex.getMessage()); + } else { + log.warn("handle subscriber error, : " + ex); + } + try { + throw ex; + } catch (IOException ex1) { + ex1.printStackTrace(); + } + } + }); + } + } + + private TaskResult getTaskResult() { + String message = ""; + try { + message = cacheUtils.popFromQueue(cachesBizConfig.getResultQueue()); + if (StringUtils.isBlank(message)) { + return null; + } + log.debug(String.format("Message: %s", message)); + return gson.fromJson(message, TaskResult.class); + } catch (Exception e) { + + if (e instanceof CacheException) { + log.warn("get message from cache exception!"); + try { + Thread.sleep(10 * 1000L); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + } else { + log.warn("serialize TaskResult error, message : " + message); + } + return null; + } + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/utils/TaskUtils.java b/juice-rest/src/main/java/com/hujiang/juice/rest/utils/TaskUtils.java new file mode 100644 index 0000000..70dc848 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/utils/TaskUtils.java @@ -0,0 +1,140 @@ +package com.hujiang.juice.rest.utils; + +import com.hujiang.juice.common.exception.RestException; +import com.hujiang.juice.common.model.Command; +import com.hujiang.juice.common.model.Constraints; +import com.hujiang.juice.common.model.Container; +import com.hujiang.juice.common.model.Resources; +import com.hujiang.juice.common.vo.SubmitTask; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.stream.Collectors; + +import static com.hujiang.juice.common.error.ErrorCode.OBJECT_INIT_ERROR; +import static com.hujiang.juice.common.error.ErrorCode.OBJECT_NOT_EQUAL_ERROR; +import static com.hujiang.juice.common.error.ErrorCode.OBJECT_NOT_NULL_ERROR; +import static com.hujiang.juice.common.model.Docker.NetWork.BRIDGE; + +/** + * Created by xujia on 16/12/5. + */ +@Slf4j +public class TaskUtils { + + public static SubmitTask.RunModel checkRunMode(SubmitTask requestTask) { + if (null == requestTask) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "request task not null!"); + } + + // check task name + if (StringUtils.isBlank(requestTask.getTaskName())) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "task name not null!"); + } + + // check & set resource + if (null == requestTask.getResources()) { + requestTask.setResources(new Resources(Resources.CPU_LEVEL.DEFAULT.getLevel(), Resources.MEM_LEVEL.DEFAULT.getLevel())); + } else { + requestTask.getResources().checkSet(); + } + + checkEnv(requestTask); + checkArgs(requestTask); + + checkConstraints(requestTask.getConstraints()); + + // check runner + if(isRunContainer(requestTask.getContainer())) { + + if(null == requestTask.getContainer().getDocker().getForcePullImage()) { + requestTask.getContainer().getDocker().setForcePullImage(true); + } + if(null == requestTask.getContainer().getDocker().getPrivileged()) { + requestTask.getContainer().getDocker().setPrivileged(false); + } + + if(StringUtils.isBlank(requestTask.getContainer().getDocker().getNet())) { + requestTask.getContainer().getDocker().setNet(BRIDGE); + } + + return SubmitTask.RunModel.CONTAINER; + } + + if(isRunCommand(requestTask.getCommands())) { + return SubmitTask.RunModel.COMMAND; + } + + // un support run model + log.warn("run mode not set, must set docker or command "); + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "must set docker or command!"); + } + + + private static boolean isRunContainer(Container container) { + if (null == container) { + return false; + } + if(null == container.getDocker() ) { + return false; + } + if (StringUtils.isBlank(container.getDocker().getImage())) { + // un support run model + log.warn("docker image not set with use container run mode"); + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(),"docker image not set with use container run mode"); + } + return true; + } + + private static boolean isRunCommand(String commands) { + if(StringUtils.isBlank(commands)){ + return false; + } + return true; + } + + + private static void checkConstraints(Constraints constraints) { + if(null == constraints) { + return; + } + + if(StringUtils.isBlank(constraints.getField())) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "constraints.field should not null!"); + } + + // to lowercase for compare + String field = constraints.getField().toLowerCase(); + if(!field.equals(Constraints.FIELD.RACK_ID.getField()) && !field.equals(Constraints.FIELD.HOSTNAME.getField())) { + throw new RestException(OBJECT_NOT_EQUAL_ERROR.getCode(), "constraints.field should be one of (rack_id, hostname)!"); + } + constraints.setField(field); + + if(null == constraints.getValues() || constraints.getValues().isEmpty()) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "constraints.values should not null!"); + } + if(constraints.getValues().stream().filter(StringUtils::isNotBlank).collect(Collectors.toSet()).size() != constraints.getValues().size()) { + throw new RestException(OBJECT_NOT_EQUAL_ERROR.getCode(), "constraints.values should not have null value when use constrains mode!"); + } + } + + private static void checkEnv(SubmitTask requestTask) { + Command.Environment env = requestTask.getEnv(); + if(env != null) { + if(StringUtils.isBlank(env.getName()) || StringUtils.isBlank(env.getValue())) { + throw new RestException(OBJECT_INIT_ERROR.getCode(), "env format error, please check env format in requestBody!"); + } + } + } + + private static void checkArgs(SubmitTask requestTask) { + List args = requestTask.getArgs(); + if(args != null && !args.isEmpty()) { + List arguments = args.stream().filter(StringUtils::isNotBlank).collect(Collectors.toList()); + if(null == arguments || arguments.isEmpty() || arguments.size() != args.size()) { + throw new RestException(OBJECT_INIT_ERROR.getCode(), "args format error, please check args format in requestBody!"); + } + } + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseAppController.java b/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseAppController.java new file mode 100644 index 0000000..9a07917 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseAppController.java @@ -0,0 +1,217 @@ +package com.hujiang.juice.rest.web.controller; + +import com.hujiang.juice.common.exception.*; +import com.hujiang.juice.common.vo.Result; +import com.hujiang.juice.common.vo.ResultBuilder; +import org.springframework.http.HttpStatus; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +import java.io.IOException; + +import static com.hujiang.juice.common.error.CommonStatusCode.REQUEST_PARAMS_INVALID_ERROR; + + +public class BaseAppController extends BaseController { + + /** + * 用于处理通用异常 + * + * @return + */ + @ResponseBody + @ExceptionHandler({ Exception.class }) + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + public Result exception(Exception e) throws IOException { + //log.warn("got a Exception",e); + + Integer status = HttpStatus.INTERNAL_SERVER_ERROR.value(); + String message = e.getMessage(); + + return handleValue(status, message); + } + + + /** + * 用于处理通用异常 + * + * @return + */ + @ResponseBody + @ExceptionHandler({ RuntimeException.class }) + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + public Result exception(RuntimeException e) throws IOException { + //log.warn("got a Exception",e); + + Integer status = HttpStatus.BAD_REQUEST.value(); + String message = e.getMessage(); + + return handleValue(status, message); + } + + /** + * 用于处理CacheException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ CacheException.class }) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public Result exception(CacheException e) throws IOException { + //log.warn("got a UnauthorizedException",e); + + String message = e.getMessage(); + Integer status = e.getCode(); + + return handleValue( status, message); + } + + + /** + * 用于处理InternalServiceException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ CommonException.class }) + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + public Result exception(CommonException e) throws IOException { + //log.warn("got a Exception",e); + + Integer status = e.getCode(); + String message = e.getMessage(); + + return handleValue(status, message); + } + + /** + * 用于处理ConfigurationException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ ConfigurationException.class }) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public Result exception(ConfigurationException e) throws IOException { + //log.warn("got a Exception",e); + + Integer status = e.getCode(); + String message = e.getMessage(); + + return handleValue(status, message); + } + + /** + * 用于处理DataBaseException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ DataBaseException.class }) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public Result exception(DataBaseException e) throws IOException { + //log.warn("got a UnauthorizedException",e); + + String message = e.getMessage(); + Integer status = e.getCode(); + + return handleValue( status, message); + } + + /** + * 用于处理InternalServiceException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ InternalServiceException.class }) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public Result exception(InternalServiceException e) throws IOException { + //log.warn("got a Exception",e); + + Integer status = e.getCode(); + String message = e.getMessage(); + + return handleValue(status, message); + } + + /** + * 用于处理RestException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ RestException.class }) + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + public Result exception(RestException e) throws IOException { + //log.warn("got a UnauthorizedException",e); + + String message = e.getMessage(); + Integer status = e.getCode(); + + return handleValue( status, message); + + } + + /** + * 用于处理UnauthorizedException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ UnauthorizedException.class }) + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + public Result exception(UnauthorizedException e) throws IOException { + //log.warn("got a UnauthorizedException",e); + + String message = e.getMessage(); + Integer status = e.getCode(); + + return handleValue( status, message); + + } + + /** + * 用于处理HttpMessageNotReadableException + * + * @param e + * @return + */ + @ResponseBody + @ExceptionHandler({ HttpMessageNotReadableException.class }) + @ResponseStatus(value = HttpStatus.BAD_REQUEST) + public Result exception(HttpMessageNotReadableException e) throws IOException { + //log.warn("got a Exception",e); + + Integer status = REQUEST_PARAMS_INVALID_ERROR.getStatus(); + String message = REQUEST_PARAMS_INVALID_ERROR.getMessage(); + + return handleValue(status, message); + } + + private Result handleValue(int status, String message) throws IOException { + return (new ResultBuilder<>()).status(status).message(message).build(); + } + protected Result buildResult(T obj) { + return (new ResultBuilder()).status(0).message("OK").data(obj).build(); + } + + protected String result(String data) throws IOException { + if(null == data) { + throw new UnauthorizedException(); + } else { + StringBuilder sb = new StringBuilder(); + sb.append("{\n").append(" \"status\" : 0,\n").append(" \"message\" : \"OK\",\n").append(" \"data\" : ").append(data).append("\n").append("}"); + return sb.toString(); + } + } +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseController.java b/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseController.java new file mode 100644 index 0000000..7d8bdc0 --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/BaseController.java @@ -0,0 +1,147 @@ +package com.hujiang.juice.rest.web.controller; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + +public class BaseController { + protected Logger logger = LoggerFactory.getLogger(getClass()); + + protected HttpServletRequest getRequest() { + HttpServletRequest request = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest(); + + return request; + } + + protected HttpServletResponse getResponse() { + HttpServletResponse response = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getResponse(); + return response; + } + + protected HttpSession getSession() { + HttpServletRequest request = getRequest(); + + HttpSession session = request.getSession(); + + return session; + } + + protected String getReqHost() { + HttpServletRequest request = getRequest(); + + String fh = request.getHeader("x-forwarded-for"); + return fh == null ? request.getRemoteHost() : fh; + } + + protected String getReqAgent() { + HttpServletRequest request = getRequest(); + + return request.getHeader("User-Agent"); + } + + protected String getReqUrl() { + HttpServletRequest request = getRequest(); + + String requestURI = request.getRequestURI(); + String contextPath = request.getContextPath(); + String path = requestURI.substring(contextPath.length()); + + return path; + } + + protected Cookie[] getCookies() { + HttpServletRequest request = getRequest(); + + return request.getCookies(); + } + + protected Object getSessionAttribute(String key) { + if (StringUtils.isBlank(key)) { + return null; + } + + HttpSession session = getSession(); + + if (null != session) { + return session.getAttribute(key); + } + + return null; + } + + protected void setSessionAttribute(String key, Object value) { + if (StringUtils.isBlank(key)) { + return; + } + + HttpSession session = getSession(); + + if (null != session) { + session.setAttribute(key, value); + } + + } + + protected void setSessionMaxInactiveInterval(int max) { + HttpSession session = getSession(); + + session.setMaxInactiveInterval(max); + } + + protected String getCookie(String key) { + if (StringUtils.isBlank(key)) { + return null; + } + + Cookie[] cookies = getCookies(); + + if ( cookies == null || cookies.length==0){ + return null; + } + + for (Cookie cookie : cookies) { + if (key.equals(cookie.getName())) { + return cookie.getValue(); + } + } + + return null; + } + + protected void addCookie(HttpServletResponse response, String key, String value, int expiry) { + if (StringUtils.isBlank(key) || null == response) { + return; + } + + Cookie cookie = new Cookie(key, value); + + cookie.setPath("/"); + cookie.setMaxAge(expiry); + + response.addCookie(cookie); + } + + /** + * 仅当前会话中有效，关闭浏览器删除Cookie + * + * @param response + * @param key + * @param value + */ + protected void addCookie(HttpServletResponse response, String key, String value) { + + addCookie(response, key, value, -1); + } + + protected void deleteCookie(HttpServletResponse response, String key) { + addCookie(response, key, null, 0); + } + +} diff --git a/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/TaskController.java b/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/TaskController.java new file mode 100644 index 0000000..04f149d --- /dev/null +++ b/juice-rest/src/main/java/com/hujiang/juice/rest/web/controller/TaskController.java @@ -0,0 +1,98 @@ +package com.hujiang.juice.rest.web.controller; + +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.common.exception.RestException; +import com.hujiang.juice.common.vo.Result; +import com.hujiang.juice.common.vo.SubmitTask; +import com.hujiang.juice.common.vo.TaskKill; +import com.hujiang.juice.common.vo.TaskReconcile; +import com.hujiang.juice.rest.service.RestService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.hujiang.juice.common.error.ErrorCode.DATA_FORMAT_ERROR; +import static com.hujiang.juice.common.error.ErrorCode.OBJECT_INIT_ERROR; +import static com.hujiang.juice.common.error.ErrorCode.OBJECT_NOT_NULL_ERROR; + + +/** + * Created by xujia on 16/12/5. + */ +@Slf4j +@RestController +@RequestMapping(value = {"/v1/tasks"}) +@CrossOrigin(origins = "*") +public class TaskController extends BaseAppController { + + @Autowired + private RestService restService; + + private Gson gson = new Gson(); + + @RequestMapping(method = RequestMethod.POST) + public Result> post(@RequestHeader(value = "X-Tenant-Id") String tenantId, @RequestBody String requestBody) throws Exception { + if(StringUtils.isBlank(requestBody)) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "requestBody is not null"); + } + log.debug("request task body --> " + requestBody); + SubmitTask object = null; + try { + object = gson.fromJson(requestBody, SubmitTask.class); + } catch (Exception e) { + throw new RestException(OBJECT_INIT_ERROR.getCode(), e.getMessage()); + } + long taskId = restService.submits(object, tenantId); + Map map = Maps.newHashMap(); + map.put("taskId", taskId); + return buildResult(map); + } + + @RequestMapping(method = RequestMethod.GET) + public Result> get(@RequestHeader(value = "X-Tenant-Id") String tenantId, @RequestParam(value = "taskIds") String taskIds) throws Exception { + if(StringUtils.isBlank(taskIds)) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "taskIds is not null"); + } + List tasks = restService.querys(tenantId, split(taskIds, ",")); + log.info("tasks : " + tasks.size()); + return buildResult(tasks); + } + + @RequestMapping(value = "/kill", method = RequestMethod.POST) + public Result kill(@RequestHeader(value = "X-Tenant-Id") String tenantId, @RequestParam(value = "taskId") Long taskId) throws Exception { + if(null == taskId) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "taskId is not null"); + } + log.info("request taskId --> " + taskId); + return buildResult(restService.kills(tenantId, taskId)); + } + + @RequestMapping(value = "/reconcile", method = RequestMethod.POST) + public Result reconcile(@RequestHeader(value = "X-Tenant-Id") String tenantId, @RequestParam(value = "taskIds") String taskIds) throws Exception { + if(StringUtils.isBlank(taskIds)) { + throw new RestException(OBJECT_NOT_NULL_ERROR.getCode(), "taskIds is not null"); + } + return buildResult(restService.reconciles(tenantId, split(taskIds, ","))); + } + + private List split(String str, String splitter) { + String[] s = str.split(splitter); + if (s.length > 0) { + try { + return Arrays.stream(s).map(String::trim).map(Long::parseLong).collect(Collectors.toList()); + } catch (Exception e) { + throw new RestException(DATA_FORMAT_ERROR.getCode(), "split string to long list error!"); + } + } + throw new RestException(DATA_FORMAT_ERROR.getCode(), + DATA_FORMAT_ERROR.getMessage() + ", taskIds error, correct taskId should be Long type & split by ','"); + } +} diff --git a/juice-rest/src/main/resources/config/application-dev.properties b/juice-rest/src/main/resources/config/application-dev.properties new file mode 100644 index 0000000..8b53574 --- /dev/null +++ b/juice-rest/src/main/resources/config/application-dev.properties @@ -0,0 +1,12 @@ +spring.datasource.url= +spring.datasource.username= +spring.datasource.password= +spring.datasource.driver-class-name=com.mysql.jdbc.Driver +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource + +spring.jooq.sql-dialect=mysql + +spring.redis.host= +spring.redis.port= +spring.redis.password= + diff --git a/juice-rest/src/main/resources/config/application.properties b/juice-rest/src/main/resources/config/application.properties new file mode 100644 index 0000000..04987e5 --- /dev/null +++ b/juice-rest/src/main/resources/config/application.properties @@ -0,0 +1,22 @@ +characterEncoding = UTF-8 + +spring.resources.cache-period=86400 + +server.compression.enabled=true +server.compression.mime-types=application/json,application/xml,text/html,text/xml,text/plain + +# SPRING MVC (HttpMapperProperties) +spring.jackson.serialization.ORDER_MAP_ENTRIES_BY_KEYS=false +spring.jackson.serialization.INDENT_OUTPUT=true +spring.mvc.date-format=yyyy-MM-dd'T'HH:mm:ssXXX + +# jackson config +spring.jackson.date-format=yyyy-MM-dd'T'HH:mm:ssXXX +spring.jackson.time-zone=Asia/Shanghai + + +logging.config=classpath:config/logback.xml +spring.profiles.active=dev + +server.port=9500 + diff --git a/juice-rest/src/main/resources/config/logback-dev.xml b/juice-rest/src/main/resources/config/logback-dev.xml new file mode 100644 index 0000000..005d02b --- /dev/null +++ b/juice-rest/src/main/resources/config/logback-dev.xml @@ -0,0 +1,58 @@ + + + + + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + + log/${APP_NAME}.log + + log/${APP_NAME}.%d{yyyy-MM-dd}.%i.log + + 365 + + + 20MB + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + + + log/ERROR.log + + log/ERROR.%d{yyyy-MM-dd}.%i.log + + 365 + + + 20MB + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + ERROR + ACCEPT + DENY + + + + + + + + + + + + + + \ No newline at end of file diff --git a/juice-rest/src/main/resources/config/logback.xml b/juice-rest/src/main/resources/config/logback.xml new file mode 100644 index 0000000..9e112d6 --- /dev/null +++ b/juice-rest/src/main/resources/config/logback.xml @@ -0,0 +1,57 @@ + + + + + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][${TASK_ID}][%5level][%logger:%L] %msg%n + + + + log/${APP_NAME}.log + + log/${APP_NAME}.%d{yyyy-MM-dd}.%i.log + + 365 + + + 20MB + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + + + log/ERROR.log + + log/ERROR.%d{yyyy-MM-dd}.%i.log + + 365 + + + 20MB + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + ERROR + ACCEPT + DENY + + + + + + + + + + + + + diff --git a/juice-service/.gitignore b/juice-service/.gitignore new file mode 100644 index 0000000..ce2fd51 --- /dev/null +++ b/juice-service/.gitignore @@ -0,0 +1,26 @@ +# Created by .ignore support plugin (hsz.mobi) +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### Example user template template +### Example user template + +# IntelliJ project files +.idea/ +.settings/ +target/ +.project +.classpath +*.iml +out +gen diff --git a/juice-service/pom.xml b/juice-service/pom.xml new file mode 100644 index 0000000..68cd884 --- /dev/null +++ b/juice-service/pom.xml @@ -0,0 +1,95 @@ + + + + juice + com.hujiang + 1.0-OPEN + + 4.0.0 + + juice-service + + + UTF-8 + 1.8 + com.hujiang.juice.service.Startup + + + + + com.hujiang + juice-common + + + com.alibaba + druid + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-core + compile + + + ch.qos.logback + logback-classic + + + + org.apache.curator + curator-recipes + + + log4j + log4j + + + + + + + juice-service + + + org.apache.maven.plugins + maven-jar-plugin + + + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + + + + maven-assembly-plugin + 2.6 + + + + ${main-class} + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/juice-service/scripts/Dockerfile b/juice-service/scripts/Dockerfile new file mode 100644 index 0000000..d0fd4e6 --- /dev/null +++ b/juice-service/scripts/Dockerfile @@ -0,0 +1,17 @@ +FROM java:8-jre-alpine + +#add timezone and default it to Shanghai +RUN apk --update add --no-cache tzdata +ENV TZ=Asia/Shanghai + +RUN mkdir -p /app/log +COPY juice-service-jar-with-dependencies.jar /app/ocs-juice.jar +COPY entrypoint.sh /app/entrypoint.sh + +RUN chmod +x /app/entrypoint.sh + +VOLUME ["/app/log"] +WORKDIR /app/ + +ENTRYPOINT ["./entrypoint.sh"] +CMD [] \ No newline at end of file diff --git a/juice-service/scripts/entrypoint.sh b/juice-service/scripts/entrypoint.sh new file mode 100644 index 0000000..57859e1 --- /dev/null +++ b/juice-service/scripts/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd /app + +echo "environment=${environment}" + +java -Dsystem.environment=${environment} -jar ocs-juice.jar \ No newline at end of file diff --git a/juice-service/src/main/java/com/hujiang/juice/service/Startup.java b/juice-service/src/main/java/com/hujiang/juice/service/Startup.java new file mode 100644 index 0000000..7ef8a62 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/Startup.java @@ -0,0 +1,45 @@ +package com.hujiang.juice.service; + +import com.hujiang.juice.service.driver.SchedulerDriver; +import com.hujiang.juice.service.utils.LogInitUtil; +import com.hujiang.juice.service.utils.zookeeper.LeaderSelectorClient; +import lombok.extern.slf4j.Slf4j; + +import static com.hujiang.juice.service.config.JUICE.*; + +/** + * Created by xujia on 16/8/9. + */ + +@Slf4j +public class Startup { + + public static void main(String[] args) { + try { + LogInitUtil.initLog(); + log.info("init juice service"); + log.info("taskQueue : " + TASK_QUEUE); + log.info("taskRetryQueue : " + TASK_RETRY_QUEUE); + log.info("resultQueue : " + TASK_RESULT_QUEUE); + log.info("managementQueue : " + MANAGEMENT_QUEUE); + SchedulerDriver schedulerDriver = new SchedulerDriver(); + if(ZOOKEEPER_DISTRIBUTE_LOCK_HA) { + LeaderSelectorClient leaderSelectorClient = null; + try { + leaderSelectorClient = new LeaderSelectorClient(schedulerDriver); + leaderSelectorClient.start(); + } catch (Exception e) { + if(null != leaderSelectorClient) { + leaderSelectorClient.close(); + } + throw e; + } + } else { + schedulerDriver.run(); + } + } catch (Exception e) { + log.error("start juice service error"); + System.exit(-1); + } + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/config/JUICE.java b/juice-service/src/main/java/com/hujiang/juice/service/config/JUICE.java new file mode 100644 index 0000000..b9f625c --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/config/JUICE.java @@ -0,0 +1,162 @@ +package com.hujiang.juice.service.config; + +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.hujiang.juice.common.utils.cache.CacheUtils; +import com.hujiang.juice.common.utils.cache.RedisCacheUtils; +import com.hujiang.juice.common.utils.db.DaoUtils; +import com.hujiang.juice.common.utils.db.JuiceDao; +import com.hujiang.juice.service.factory.JuiceFactory; +import com.hujiang.juice.service.factory.DataSourceFactory; +import com.hujiang.juice.service.factory.RedisFactory; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * Created by xujia on 16/9/27. + */ +@Slf4j +public class JUICE { + + public static final Gson gson = new Gson(); + // retry queue every offer + public static final int CACHE_TRIES = 5; + + // stream-id + public static final String STREAM_ID = "Mesos-Stream-Id"; + // framework failover timeout + public static final double FRAMEWORK_FAILOVER_TIMEOUT = 30 * 24 * 60 * 60; + + public static final String HTTP_SEPERATOR = "/"; + + public static final String ZKLOCKS = "/mesos/zklocks"; + + + // for curator use + private static final String CONFIG_ZOOKEEPER_DISTRIBUTE_LOCK_HA = "zookeeper.distribute.lock.ha"; + private static final boolean DEFAULT_ZOOKEEPER_DISTRIBUTE_LOCK_HA = true; + public static final boolean ZOOKEEPER_DISTRIBUTE_LOCK_HA = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_ZOOKEEPER_DISTRIBUTE_LOCK_HA, DEFAULT_ZOOKEEPER_DISTRIBUTE_LOCK_HA); + + // for curator use + private static final String CONFIG_MESOS_ROOT_PATH = "mesos.root.path"; + private static final String DEFAULT_MESOS_ROOT_PATH = "/mesos"; + public static final String MESOS_ROOT_PATH = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_ROOT_PATH, DEFAULT_MESOS_ROOT_PATH); + + private static final String CONFIG_DEFAULT_PATH_NAME = "mesos.default.path"; + public static final String DEFAULT_PATH_NAME = "json.info"; + public static final String PATH_NAME = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_DEFAULT_PATH_NAME, DEFAULT_PATH_NAME); + + // mesos scheduler name + private static final String CONFIG_MESOS_SCHEDULER_NAME = "mesos.scheduler.name"; + private static final String DEFAULT_MESOS_SCHEDULER_NAME = "juice-service-" + JuiceFactory.INSTANCE.getEnv(); + public static final String MESOS_SCHEDULER_NAME = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_SCHEDULER_NAME, DEFAULT_MESOS_SCHEDULER_NAME); + +// only support zk start mode, so do not use MESOS_SCHEDULER_END_POINT to start service +// // mesos framework end point by uri (only start with uri mode should be configured） +// private static final String CONFIG_MESOS_SCHEDULER_END_POINT = "mesos.framework.uri"; +// public static final String MESOS_SCHEDULER_END_POINT = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_SCHEDULER_END_POINT, ""); + + // discover mesos framework end point by zk (general use) + private static final String CONFIG_MESOS_SCHEDULER_END_POINT_ZK = "mesos.framework.zk"; + public static final String MESOS_SCHEDULER_END_POINT_ZK = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_SCHEDULER_END_POINT_ZK, ""); + + // mesos framework role + private static final String CONFIG_MESOS_FRAMEWORK_ROLE = "mesos.framework.role"; + public static final String MESOS_FRAMEWORK_ROLE = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_FRAMEWORK_ROLE, "*"); + + // resources can only use threshold + private static final String CONFIG_RESOURCES_USE_THRESHOLD = "resources.use.threshold"; + private static final double DEFAULT_RESOURCES_USE_THRESHOLD = 0.8; + public static final double RESOURCES_USE_THRESHOLD = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_RESOURCES_USE_THRESHOLD, DEFAULT_RESOURCES_USE_THRESHOLD); + + // mesos framework attrs + private static final String CONFIG_MESOS_FRAMEWORK_ATTR_FILTER = "mesos.framework.attr"; + private static final String MESOS_FRAMEWORK_ATTR_FILTER = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_FRAMEWORK_ATTR_FILTER); + public static final Map MESOS_FRAMEWORK_ATTR_FILTER_MAP = getAttrFilterMap(",", "\\|"); + public static final BitSet MESOS_FRAMEWORK_ATTR_FILTER_DEFAULT_BITSET = getdefaultBitSet(","); + + // offer allocation send pool + private static final String CONFIG_OFFER_SEND_POOL = "send.pool.size"; + private static final int DEFAULT_OFFER_SEND_POOL = 20; + public static final int OFFER_SEND_POOL = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_OFFER_SEND_POOL, DEFAULT_OFFER_SEND_POOL); + + // offer auxiliary handler pool + private static final String CONFIG_AUXILIARY_POOL = "auxiliary.pool.size"; + private static final int DEFAULT_AUXILIARY_POOL = 20; + public static final int AUXILIARY_POOL = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_AUXILIARY_POOL, DEFAULT_AUXILIARY_POOL); + + // task max reserved times + private static final String CONFIG_MAX_RESERVED = "max.reserved"; + private static final int DEFAULT_MAX_RESERVED = 1024; + public static final int MAX_RESERVED = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MAX_RESERVED, DEFAULT_MAX_RESERVED); + + // framework tag + private static final String CONFIG_MESOS_FRAMEWORK_TAG = "mesos.framework.tag"; + private static final String DEFAULT_MESOS_FRAMEWORK_TAG = "mesos.framework.tag." + JuiceFactory.INSTANCE.getEnv(); + public static final String MESOS_FRAMEWORK_TAG = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_MESOS_FRAMEWORK_TAG, DEFAULT_MESOS_FRAMEWORK_TAG); + + // juice task queue name + private static final String CONFIG_TASK_QUEUE = "juice.task.queue"; + private static final String DEFAULT_TASK_QUEUE = "juice.task.queue." + JuiceFactory.INSTANCE.getEnv(); + public static final String TASK_QUEUE = RedisFactory.INSTANCE.getConfigure().getValue(CONFIG_TASK_QUEUE, DEFAULT_TASK_QUEUE); + + // juice task retry queue name + private static final String CONFIG_TASK_RETRY_QUEUE = "juice.task.retry.queue"; + private static final String DEFAULT_TASK_RETRY_QUEUE = "juice.task.retry.queue." + JuiceFactory.INSTANCE.getEnv(); + public static final String TASK_RETRY_QUEUE = RedisFactory.INSTANCE.getConfigure().getValue(CONFIG_TASK_RETRY_QUEUE, DEFAULT_TASK_RETRY_QUEUE); + + // juice task result queue + private static final String CONFIG_RESULT_QUEUE = "juice.task.result.queue"; + private static final String DEFAULT_RESULT_QUEUE = "juice.task.result.queue." + JuiceFactory.INSTANCE.getEnv(); + public static final String TASK_RESULT_QUEUE = RedisFactory.INSTANCE.getConfigure().getValue(CONFIG_RESULT_QUEUE, DEFAULT_RESULT_QUEUE); + + // juice management queue + private static final String CONFIG_MANAGEMENT_QUEUE = "juice.management.queue"; + private static final String DEFAULT_MANAGEMENT_QUEUE = "juice.management.queue." + JuiceFactory.INSTANCE.getEnv(); + public static String MANAGEMENT_QUEUE = RedisFactory.INSTANCE.getConfigure().getValue(CONFIG_MANAGEMENT_QUEUE, DEFAULT_MANAGEMENT_QUEUE); + + // juice task expire seconds in queue + private static final String CONFIG_TASK_RETRY_EXPIRE_TIME = "task.retry.expire.time"; + private static final int DEFAULT_TASK_RETRY_EXPIRE_TIME = 86400; + public static final long TASK_RETRY_EXPIRE_TIME = JuiceFactory.INSTANCE.getConfigure().getValue(CONFIG_TASK_RETRY_EXPIRE_TIME, DEFAULT_TASK_RETRY_EXPIRE_TIME); + + // cache + public static final CacheUtils cacheUtils = new RedisCacheUtils(RedisFactory.Redis.INSTANCE.redisUtil()); + // dao + public static final DaoUtils daoUtils = new DaoUtils(new JuiceDao(DataSourceFactory.JOOQ.INSTANCE.getContext())); + + private static Map getAttrFilterMap(@NotNull String spilt, @NotNull String cut) { + + AtomicInteger i = new AtomicInteger(0); + + Map valueMap = Maps.newHashMap(); + Arrays.stream(MESOS_FRAMEWORK_ATTR_FILTER.split(spilt)).forEach( + v -> { + final int step = i.getAndIncrement(); + Arrays.stream(v.split(cut)).forEach( + v1 -> { + valueMap.put(v1, step); + } + ); + } + ); + + return valueMap; + } + + private static BitSet getdefaultBitSet(@NotNull String spilt) { + BitSet bs = new BitSet(); + if(StringUtils.isNotBlank(MESOS_FRAMEWORK_ATTR_FILTER)) { + int length = MESOS_FRAMEWORK_ATTR_FILTER.split(spilt).length; + for (int i = 0; i < length; i++) { + bs.set(i); + } + } + return bs; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/driver/SchedulerDriver.java b/juice-service/src/main/java/com/hujiang/juice/service/driver/SchedulerDriver.java new file mode 100644 index 0000000..e3588fb --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/driver/SchedulerDriver.java @@ -0,0 +1,286 @@ +package com.hujiang.juice.service.driver; + +import com.google.common.collect.Maps; +import com.hujiang.juice.common.exception.CacheException; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.service.exception.UnrecoverException; +import com.hujiang.juice.service.support.Support; +import com.hujiang.juice.common.vo.TaskResult; +import com.hujiang.juice.service.exception.DriverException; +import com.hujiang.juice.service.model.Host; +import com.hujiang.juice.service.model.SchedulerCalls; +import com.hujiang.juice.service.service.AuxiliaryService; +import com.hujiang.juice.service.service.SchedulerService; +import com.hujiang.juice.service.utils.SendUtils; +import com.hujiang.juice.service.utils.protocol.Protobuf; +import com.hujiang.juice.service.utils.protocol.Protocol; +import com.hujiang.juice.service.utils.zookeeper.CuratorUtils; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import okhttp3.Response; +import org.apache.commons.lang3.StringUtils; +import org.apache.mesos.v1.scheduler.Protos; + +import java.io.InputStream; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Lists.newArrayList; +import static com.hujiang.juice.common.vo.TaskResult.Result.ERROR; +import static com.hujiang.juice.service.config.JUICE.*; +import static org.apache.mesos.v1.Protos.*; + +/** + * Created by xujia on 16/11/22. + */ + + +@Slf4j +@Data +@EqualsAndHashCode(callSuper = false) +public class SchedulerDriver { + private FrameworkID frameworkId; + private CuratorUtils curatorUtils; + private volatile Host host = new Host(); + + private Support support; + private Protocol protocol; + private volatile String streamId; + private final List declines = newArrayList(); + private final Map> attrMap = Maps.newHashMap(); + + private final Map killMap = Maps.newConcurrentMap(); + + public String getUrl() { + return host.getUrl(); + } + + + public SchedulerDriver() { + protocol = new Protobuf(); + support = new Support(MESOS_FRAMEWORK_ROLE); + if (StringUtils.isNotBlank(MESOS_SCHEDULER_END_POINT_ZK)) { + curatorUtils = new CuratorUtils(MESOS_SCHEDULER_END_POINT_ZK, host); + curatorUtils.init(); + log.info("service is initializing and running by curator(zookeeper)!"); + } else { + log.error("mesos host is not config, service will down!"); + System.exit(-1); + } + log.info("host : " + host.getHost()); + } + + public void run() { + + // get framework id + frameworkId = SchedulerService.getFrameworkId(); + + // start a new thread to listen task management list + log.info("start juice auxiliary service"); + AuxiliaryService.start(this); + + try { + log.info("start juice service"); + while (true) { + try { + connecting(); + } catch (Exception e) { + if (e instanceof UnrecoverException) { + log.error("server will recover now, cause : " + e); + reset(((UnrecoverException) e).isResetFrameworkId()); + } + + log.error("server will restart after 30s due to : " + e); + try { + Thread.sleep(30 * 1000L); + } catch (InterruptedException e1) { + log.warn(e1.getMessage()); + } + } + } + } catch (Exception e) { + log.error(e.getMessage()); + } + } + + + private void connecting() throws Exception { + InputStream stream = null; + Response res = null; + + try { + Protos.Call call = subscribeCall(); + res = Restty.create(getUrl()) + .addAccept(protocol.mediaType()) + .addMediaType(protocol.mediaType()) + .addKeepAlive() + .requestBody(protocol.getSendBytes(call)) + .post(); + + streamId = res.header(STREAM_ID); + stream = res.body().byteStream(); + log.info("send subscribe, frameworkId : " + frameworkId + " , url " + getUrl() + ", streamId : " + streamId); + log.debug("subscribe call : " + call); + if (null == stream) { + log.warn("stream is null"); + throw new DriverException("stream is null"); + } + while (true) { + int size = SendUtils.readChunkSize(stream); + byte[] event = SendUtils.readChunk(stream, size); + + onEvent(event); + } + } catch (Exception e) { + log.error("service handle error, due to : " + e); + throw e; + } finally { + if (null != stream) { + stream.close(); + } + if (null != res) { + res.close(); + } + streamId = null; + } + } + + private void reset(boolean isResetFrameworkId) { + if (isResetFrameworkId) { + String removingId = frameworkId.getValue(); + try { + killMap.clear(); + declines.clear(); + attrMap.clear(); + AuxiliaryService.loggedErrors(); + AuxiliaryService.getSendErrors().clear(); + SchedulerService.removeFrameworkId(); + } catch (CacheException ex) { + log.error("remove framework id : " + removingId + " from db error!"); + } + AuxiliaryService.loggedErrors(); + frameworkId = SchedulerService.genFrameworkId(); + } + } + + private Protos.Call subscribeCall() { + + String hostName = System.getenv("HOST"); + if (StringUtils.isBlank(hostName)) { + try { + hostName = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + hostName = "unknown host"; + e.printStackTrace(); + } + } + + return SchedulerCalls.subscribe( + FrameworkInfo.newBuilder() + .setId(frameworkId) + .setHostname(hostName) + .setUser(Optional.ofNullable(System.getenv("user")).orElse("root")) + .setName(MESOS_SCHEDULER_NAME) + .setFailoverTimeout(FRAMEWORK_FAILOVER_TIMEOUT) + .setRole(MESOS_FRAMEWORK_ROLE) + .build()); + } + + private void onEvent(byte[] bytes) { + + Protos.Event event = null; + try { + event = (Protos.Event) protocol.getEvent(bytes, Protos.Event.class); + } catch (Exception e) { + log.warn("parser event error, raw event : " + new String(bytes)); + throw new DriverException("parser event error!"); + } + + log.debug("event type : " + event.getType()); + switch (event.getType()) { + case SUBSCRIBED: + Protos.Event.Subscribed subscribed = event.getSubscribed(); + SchedulerService.subscribed(subscribed, frameworkId); + break; + case OFFERS: + try { + long start = System.currentTimeMillis(); + event.getOffers().getOffersList().stream() + .filter(of -> { + if (SchedulerService.filterAndAddAttrSys(of, attrMap)) { + return true; + } + declines.add(of.getId()); + return false; + }) + .forEach( + of -> { + List tasks = newArrayList(); + String offerId = of.getId().getValue(); + try { + SchedulerService.handleOffers(killMap, support, of, attrMap.get(offerId), declines, tasks); + } catch (Exception e) { + declines.add(of.getId()); + tasks.forEach( + t -> { + AuxiliaryService.getTaskErrors() + .push(new TaskResult(com.hujiang.juice.common.model.Task.splitTaskNameId(t.getTaskId().getValue()) + , ERROR, "task failed due to exception!")); + } + ); + tasks.clear(); + } + if (tasks.size() > 0) { + AuxiliaryService.acceptOffer(protocol, streamId, of.getId(), frameworkId, tasks, getUrl()); + } + } + ); + + if (declines.size() > 0) { + AuxiliaryService.declineOffer(protocol, streamId, frameworkId, SchedulerCalls.decline(frameworkId, declines), getUrl()); + } + long end = System.currentTimeMillis(); + log.debug("accept --> used time : " + (end - start) + " ms"); + } finally { + declines.clear(); + attrMap.clear(); + } + break; + case UPDATE: + TaskStatus status = event.getUpdate().getStatus(); + if (status.hasUuid()) { + SchedulerService.update(killMap, status, protocol, frameworkId, streamId, getUrl()); + try { + SendUtils.sendCall(acknowledgeCall(status), protocol, streamId, getUrl()); + } catch (Exception e) { + log.warn("send acknowledge call error!"); + throw new DriverException(e); + } + } + break; + case MESSAGE: + Protos.Event.Message message = event.getMessage(); + SchedulerService.message(message.getAgentId(), message.getData().toByteArray()); + break; + case ERROR: + SchedulerService.error(event); + case RESCIND: + case FAILURE: + case HEARTBEAT: + break; // ignore + default: + log.warn("Unsupported event : " + event); + throw new DriverException("Unsupported event : " + event); + } + } + + + private Protos.Call acknowledgeCall(TaskStatus status) { + return SchedulerCalls.ackUpdate(frameworkId, status.getUuid(), status.getAgentId(), status.getTaskId()); + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/exception/DriverException.java b/juice-service/src/main/java/com/hujiang/juice/service/exception/DriverException.java new file mode 100644 index 0000000..33b89f8 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/exception/DriverException.java @@ -0,0 +1,18 @@ +package com.hujiang.juice.service.exception; + +/** + * Created by xujia on 16/12/19. + */ +public class DriverException extends RuntimeException{ + public DriverException(String message) { + super(message); + } + + public DriverException(Throwable cause) { + super(cause); + } + + public DriverException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/exception/UnrecoverException.java b/juice-service/src/main/java/com/hujiang/juice/service/exception/UnrecoverException.java new file mode 100644 index 0000000..e20217f --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/exception/UnrecoverException.java @@ -0,0 +1,23 @@ +package com.hujiang.juice.service.exception; + +import lombok.Data; + +/** + * Created by xujia on 16/11/21. + */ + +@Data +public class UnrecoverException extends RuntimeException { + + private boolean isResetFrameworkId = false; + + public UnrecoverException(String message, boolean isResetFrameworkId) { this(message, null, isResetFrameworkId ); } + + public UnrecoverException(Throwable cause, boolean isResetFrameworkId) { this(null, cause, isResetFrameworkId); } + + public UnrecoverException(String message, Throwable cause, boolean isResetFrameworkId) { + super(message, cause); + this.isResetFrameworkId = isResetFrameworkId; + } + +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/factory/DataSourceFactory.java b/juice-service/src/main/java/com/hujiang/juice/service/factory/DataSourceFactory.java new file mode 100644 index 0000000..323c16f --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/factory/DataSourceFactory.java @@ -0,0 +1,128 @@ +package com.hujiang.juice.service.factory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.hujiang.juice.common.config.Configure; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.exception.ConfigurationException; +import org.apache.commons.lang3.StringUtils; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DefaultDSLContext; + +import javax.sql.DataSource; +import java.io.FileNotFoundException; + +/** + * Created by xujia on 17/3/4. + */ +public enum DataSourceFactory { + + INSTANCE; + + private Configure configure; + + DataSourceFactory() { + try { + String env = System.getProperty("system.environment"); + if(StringUtils.isBlank(env)) { + throw new ConfigurationException(ErrorCode.SYSTEM_ENV_NOT_VALID.getCode(), "can't get env, service stop!"); + } + configure = new Configure("application-" + env); + } catch (FileNotFoundException e) { + e.printStackTrace(); + System.exit(1); + } + } + + //database config + public static final String CONFIG_DATABASE_DRIVER = "db.driver"; + public static final String CONFIG_DATABASE_URL = "db.url"; + public static final String CONFIG_DATABASE_USER = "db.user"; + public static final String CONFIG_DATABASE_PASSWORD = "db.password"; + public static final String CONFIG_DATABASE_POOL_INITIAL_SIZE = "db.pool.initial.size"; + public static final String CONFIG_DATABASE_POOL_MAX_ACTIVE = "db.pool.max.active"; + public static final String CONFIG_DATABASE_POOL_VALIDATION_QUERY = "db.pool.validation.query"; + public static final String CONFIG_DATABASE_POOL_TEST_WHILE_IDLE = "db.pool.test.while.idle"; + public static final String CONFIG_DATABASE_POOL_TIME_BETWEEN_EVICTION_RUNS_MILLS = "db.pool.time.between.eviction.runs.millis"; + + //database default value + public static final int DEFAULT_DATABASE_POOL_INITIAL_SIZE = 5; + public static final int DEFAULT_DATABASE_POOL_MAX_ACTIVE = 10; + public static final String DEFAULT_DATABASE_POOL_VALIDATION_QUERY = "select 1"; + public static final boolean DEFAULT_DATABASE_POOL_TEST_WHILE_IDLE = true; + public static final int DEFAULT_DATABASE_POOL_TIME_BETWEEN_EVICTION_RUNS_MILLS = 300000; + + public String getValue(String name) { + return configure.getValue(name); + } + + public String getValue(String name, String defaultValue) { + String value = getValue(name); + if (value == null) { + return defaultValue; + } + return value; + } + + public int getValue(String name, int defaultValue) { + return configure.getValue(name, defaultValue); + } + + public long getValue(String name, long defaultValue) { + return configure.getValue(name, defaultValue); + } + + public float getValue(String name, float defaultValue) { + return configure.getValue(name, defaultValue); + } + + public double getValue(String name, double defaultValue) { + return configure.getValue(name, defaultValue); + } + + public boolean getValue(String name, boolean defaultValue) { + return configure.getValue(name, defaultValue); + } + + public Configure getConfigure(){ + return configure; + } + + public enum Database { + INSTANCE(DataSourceFactory.INSTANCE); + + private DataSource dataSource; + + Database(DataSourceFactory configure) { + DruidDataSource dataSource = new DruidDataSource(); + dataSource.setDriverClassName(configure.getValue(CONFIG_DATABASE_DRIVER)); + dataSource.setUrl(configure.getValue(CONFIG_DATABASE_URL)); + dataSource.setUsername(configure.getValue(CONFIG_DATABASE_USER)); + dataSource.setPassword(configure.getValue(CONFIG_DATABASE_PASSWORD)); + + dataSource.setInitialSize(configure.getValue(CONFIG_DATABASE_POOL_INITIAL_SIZE, DEFAULT_DATABASE_POOL_INITIAL_SIZE)); + dataSource.setMaxActive(configure.getValue(CONFIG_DATABASE_POOL_MAX_ACTIVE, DEFAULT_DATABASE_POOL_MAX_ACTIVE)); + dataSource.setValidationQuery(configure.getValue(CONFIG_DATABASE_POOL_VALIDATION_QUERY, DEFAULT_DATABASE_POOL_VALIDATION_QUERY)); + dataSource.setTestWhileIdle(configure.getValue(CONFIG_DATABASE_POOL_TEST_WHILE_IDLE, DEFAULT_DATABASE_POOL_TEST_WHILE_IDLE)); + dataSource.setTimeBetweenEvictionRunsMillis(configure.getValue(CONFIG_DATABASE_POOL_TIME_BETWEEN_EVICTION_RUNS_MILLS, DEFAULT_DATABASE_POOL_TIME_BETWEEN_EVICTION_RUNS_MILLS)); + this.dataSource = dataSource; + } + + public DataSource getDataSource() { + return dataSource; + } + } + + public enum JOOQ{ + INSTANCE(Database.INSTANCE.getDataSource()); + + private DSLContext context; + JOOQ(DataSource dataSource){ + context = new DefaultDSLContext(dataSource, SQLDialect.MYSQL); + } + + public DSLContext getContext() { + return context; + } + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/factory/JuiceFactory.java b/juice-service/src/main/java/com/hujiang/juice/service/factory/JuiceFactory.java new file mode 100644 index 0000000..da03254 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/factory/JuiceFactory.java @@ -0,0 +1,43 @@ +package com.hujiang.juice.service.factory; + +import com.hujiang.juice.common.config.Configure; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.FileNotFoundException; + +/** + * Created by xujia on 16/11/3. + */ + +@Slf4j +public enum JuiceFactory{ + + INSTANCE; + + private Configure configure; + private String env; + + JuiceFactory(){ + try { + env = System.getProperty("system.environment"); + if(StringUtils.isBlank(env)) { + throw new RuntimeException("can't get env, service stop!"); + } + configure = new Configure("application-" + env); + + } catch (FileNotFoundException e) { + + System.err.println(e.getMessage()); + System.exit(1); + } + } + + public String getEnv() { + return env; + } + + public Configure getConfigure() { + return configure; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/factory/RedisFactory.java b/juice-service/src/main/java/com/hujiang/juice/service/factory/RedisFactory.java new file mode 100644 index 0000000..3a44a98 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/factory/RedisFactory.java @@ -0,0 +1,72 @@ +package com.hujiang.juice.service.factory; + + +import com.hujiang.juice.common.config.Configure; +import com.hujiang.juice.common.error.ErrorCode; +import com.hujiang.juice.common.exception.CacheException; +import com.hujiang.juice.common.exception.ConfigurationException; +import com.hujiang.juice.common.utils.cache.RedisUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.FileNotFoundException; + + + +/** + * Created by xujia on 16/12/5. + */ + +@Slf4j +public enum RedisFactory { + + INSTANCE; + + private Configure configure; + + RedisFactory() { + try { + String env = System.getProperty("system.environment"); + if (StringUtils.isBlank(env)) { + throw new ConfigurationException(ErrorCode.SYSTEM_ENV_NOT_VALID.getCode(), "can't get env, service stop!"); + } + configure = new Configure("application-" + env); + } catch (FileNotFoundException e) { + e.printStackTrace(); + System.exit(1); + } + } + + public String getValue(String name) { + return configure.getValue(name); + } + + public Configure getConfigure() { + return configure; + } + + // redis connection config + public static final String CONFIG_REDIS_HOST = "redis.host"; + public static final String CONFIG_REDIS_PORT = "redis.port"; + public static final String CONFIG_REDIS_PASSWD = "redis.password"; + + public enum Redis { + INSTANCE(RedisFactory.INSTANCE); + + private RedisUtil redisUtil; + + Redis(RedisFactory redisFactory) { + redisUtil = new RedisUtil(redisFactory.getValue(CONFIG_REDIS_HOST), + Integer.parseInt(redisFactory.getValue(CONFIG_REDIS_PORT)), + StringUtils.isNotBlank(redisFactory.getValue(CONFIG_REDIS_PASSWD)) ? redisFactory.getValue(CONFIG_REDIS_PASSWD) : null); + } + + public RedisUtil redisUtil() { + if (null == redisUtil) { + throw new CacheException(ErrorCode.REDIS_INIT_ERROR.getCode(), "init redisUtil failed"); + } + return redisUtil; + } + } +} + diff --git a/juice-service/src/main/java/com/hujiang/juice/service/model/Host.java b/juice-service/src/main/java/com/hujiang/juice/service/model/Host.java new file mode 100644 index 0000000..2a168c6 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/model/Host.java @@ -0,0 +1,17 @@ +package com.hujiang.juice.service.model; + +import lombok.Data; + +/** + * Created by xujia on 17/2/6. + */ + +@Data +public class Host { + private String host; + private static final String uri = "/api/v1/scheduler"; + + public String getUrl() { + return host + uri; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/model/SchedulerCalls.java b/juice-service/src/main/java/com/hujiang/juice/service/model/SchedulerCalls.java new file mode 100644 index 0000000..05da4af --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/model/SchedulerCalls.java @@ -0,0 +1,159 @@ +package com.hujiang.juice.service.model; + +import com.google.protobuf.ByteString; +import com.hujiang.juice.common.model.TaskManagement; +import org.jetbrains.annotations.NotNull; + +import java.util.List; + +import static org.apache.mesos.v1.Protos.*; +import static org.apache.mesos.v1.scheduler.Protos.Call; + +/** + * Created by xujia on 16/11/22. + */ +public class SchedulerCalls { + + private SchedulerCalls() {} + + @NotNull + public static Call kill(@NotNull final FrameworkID frameworkId, + @NotNull final TaskManagement.TaskAgentRel taskAgentRel + ) { + return Call.newBuilder() + .setFrameworkId(frameworkId) + .setType(Call.Type.KILL) + .setKill( + Call.Kill.newBuilder() + .setTaskId(taskAgentRel.protosTaskId()) + .setAgentId(taskAgentRel.protosAgentId()) + .build() + ).build(); + } + + @NotNull + public static Call reconcile(@NotNull final FrameworkID frameworkId, + @NotNull final List taskAgentRels + ) { + Call.Reconcile.Builder builder = Call.Reconcile.newBuilder(); + taskAgentRels.stream().parallel().forEach( + taskAgentRel -> { + builder.addTasks(Call.Reconcile.Task.newBuilder() + .setAgentId(taskAgentRel.protosAgentId()) + .setTaskId(taskAgentRel.protosTaskId()) + .build()); + } + ); + + return Call.newBuilder() + .setFrameworkId(frameworkId) + .setType(Call.Type.RECONCILE) + .setReconcile(builder) + .build(); + } + + @NotNull + public static Call tearDown(@NotNull final FrameworkID frameworkId){ + return Call.newBuilder() + .setFrameworkId(frameworkId) + .setType(Call.Type.TEARDOWN) + .build(); + } + @NotNull + public static Call ackUpdate( + @NotNull final FrameworkID frameworkId, + @NotNull final ByteString uuid, + @NotNull final AgentID agentId, + @NotNull final TaskID taskId + ) { + return Call.newBuilder() + .setFrameworkId(frameworkId) + .setType(Call.Type.ACKNOWLEDGE) + .setAcknowledge( + Call.Acknowledge.newBuilder() + .setUuid(uuid) + .setAgentId(agentId) + .setTaskId(taskId) + .build() + ) + .build(); + } + + + public static @NotNull + Call accept( + final @NotNull FrameworkID frameworkId, + final @NotNull OfferID offerId, + @NotNull final List tasks + ) { + return Call.newBuilder() + .setFrameworkId(frameworkId) + .setType(Call.Type.ACCEPT) + .setAccept( + Call.Accept.newBuilder() + .addOfferIds(offerId) + .addOperations( + Offer.Operation.newBuilder() + .setType(Offer.Operation.Type.LAUNCH) + .setLaunch( + Offer.Operation.Launch.newBuilder() + .addAllTaskInfos(tasks) + ) + ) + ) + .build(); + } + + @NotNull + public static Call decline(@NotNull final FrameworkID frameworkId, @NotNull final List offerIds) { + return Call.newBuilder() + .setFrameworkId(frameworkId) + .setType(Call.Type.DECLINE) + .setDecline( + Call.Decline.newBuilder() + .addAllOfferIds(offerIds) + ) + .build(); + } + + @NotNull + public static Call subscribe( + @NotNull final String frameworkId, + @NotNull final String user, + @NotNull final String frameworkName, + final long failoverTimeoutSeconds + ) { + final FrameworkID frameworkID = FrameworkID.newBuilder().setValue(frameworkId).build(); + return subscribe(frameworkID, user, frameworkName, failoverTimeoutSeconds); + } + + @NotNull + public static Call subscribe( + @NotNull final FrameworkID frameworkId, + @NotNull final String user, + @NotNull final String frameworkName, + final long failoverTimeoutSeconds + ) { + final FrameworkInfo frameworkInfo = FrameworkInfo.newBuilder() + .setId(frameworkId) + .setUser(user) + .setName(frameworkName) + .setFailoverTimeout(failoverTimeoutSeconds) + .build(); + return subscribe(frameworkInfo); + } + + @NotNull + public static Call subscribe( + @NotNull final FrameworkInfo frameworkInfo + ) { + return Call.newBuilder() + .setFrameworkId(frameworkInfo.getId()) + .setType(Call.Type.SUBSCRIBE) + .setSubscribe( + Call.Subscribe.newBuilder() + .setFrameworkInfo(frameworkInfo) + ) + .build(); + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/model/SendErrors.java b/juice-service/src/main/java/com/hujiang/juice/service/model/SendErrors.java new file mode 100644 index 0000000..ef1ee54 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/model/SendErrors.java @@ -0,0 +1,22 @@ +package com.hujiang.juice.service.model; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.mesos.v1.Protos; + + +/** + * Created by xujia on 17/3/4. + */ + +@Slf4j +@Data +public class SendErrors { + private Protos.FrameworkID frameworkId; + private org.apache.mesos.v1.scheduler.Protos.Call call; + + public SendErrors(Protos.FrameworkID frameworkId, org.apache.mesos.v1.scheduler.Protos.Call call) { + this.frameworkId = frameworkId; + this.call = call; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/package-info.java b/juice-service/src/main/java/com/hujiang/juice/service/package-info.java new file mode 100644 index 0000000..ce303de --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/package-info.java @@ -0,0 +1 @@ +package com.hujiang.juice.service; \ No newline at end of file diff --git a/juice-service/src/main/java/com/hujiang/juice/service/service/AuxiliaryService.java b/juice-service/src/main/java/com/hujiang/juice/service/service/AuxiliaryService.java new file mode 100644 index 0000000..d01726e --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/service/AuxiliaryService.java @@ -0,0 +1,348 @@ +package com.hujiang.juice.service.service; + +import com.google.common.collect.Lists; +import com.hujiang.jooq.juice.tables.pojos.JuiceTask; +import com.hujiang.juice.common.exception.CacheException; +import com.hujiang.juice.common.exception.DataBaseException; +import com.hujiang.juice.common.model.*; +import com.hujiang.juice.common.vo.TaskResult; +import com.hujiang.juice.service.driver.SchedulerDriver; +import com.hujiang.juice.service.exception.DriverException; +import com.hujiang.juice.service.model.SchedulerCalls; +import com.hujiang.juice.service.model.SendErrors; +import com.hujiang.juice.service.utils.SendUtils; +import com.hujiang.juice.service.utils.protocol.Protocol; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.mesos.v1.scheduler.Protos; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; + +import static com.hujiang.juice.common.config.COMMON.KILL; +import static com.hujiang.juice.common.config.COMMON.RECONCILE; +import static com.hujiang.juice.common.utils.CommonUtils.currentTimeSeconds; +import static com.hujiang.juice.service.config.JUICE.*; + +import static org.apache.mesos.v1.Protos.*; + +/** + * Created by xujia on 17/1/18. + */ + +@Slf4j +public class AuxiliaryService { + + private static final ExecutorService fixedSendPool = Executors.newFixedThreadPool(OFFER_SEND_POOL); + private static final ExecutorService fixedAuxiliaryPool = Executors.newFixedThreadPool(AUXILIARY_POOL); + private static final LinkedBlockingDeque sendErrors = new LinkedBlockingDeque<>(100000); + private static final LinkedBlockingDeque taskErrors = new LinkedBlockingDeque<>(100000); + + public static LinkedBlockingDeque getSendErrors() { + return sendErrors; + } + + public static LinkedBlockingDeque getTaskErrors() { + return taskErrors; + } + + public static void start(SchedulerDriver schedulerDriver) { + + // one thread to consumer + fixedAuxiliaryPool.submit(() -> consumer(schedulerDriver)); + + // one thread to schedule update freamework id + fixedAuxiliaryPool.submit(() -> scheduledFramework(schedulerDriver)); + + + //one thread to handle un-handle-task-list + fixedAuxiliaryPool.submit(AuxiliaryService::handleTaskErrors); + + //one thread to handle offer-task send errors + fixedAuxiliaryPool.submit(() -> handleSendErrors(schedulerDriver)); + } + + public static void updateTask(@NotNull Map killMap, long taskId, String agentId, boolean isToKilled) { + fixedAuxiliaryPool.submit(() -> { + try { + if (!isToKilled) { + daoUtils.updateTask(taskId, agentId); + } else { + daoUtils.finishTask(taskId, TaskResult.Result.KILLED.getType(), "task not run due to killed"); + killMap.remove(taskId); + } + } catch (DataBaseException e) { + log.error("database operation error, update task agent rel in db failed, it will influence kill task model, taskId : " + taskId + ", agentId :" + agentId); + } + }); + } + + public static void acceptOffer(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull OfferID offerID, + final @NotNull FrameworkID frameworkID, final @NotNull List taskInfos, + final @NotNull String url) { + fixedSendPool.submit(() -> accept(protocol, streamId, offerID, frameworkID, taskInfos, url)); + } + + + public static void declineOffer(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull FrameworkID frameworkID, + final @NotNull Protos.Call call, final @NotNull String url) { + fixedSendPool.submit(() -> decline(protocol, streamId, frameworkID, call, url)); + } + + + public static void loggedErrors() { + if (taskErrors.size() > 0) { + log.error("logged unhandle error list : "); + taskErrors.stream().parallel().forEach( + un -> { + log.error("record : " + gson.toJson(un)); + } + ); + taskErrors.clear(); + } + } + + public static void consumer(SchedulerDriver schedulerDriver) { + while (true) { + try { + String message = cacheUtils.popFromQueue(MANAGEMENT_QUEUE); + if (StringUtils.isBlank(message)) { + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + e.printStackTrace(); + } + continue; + } + + log.debug(String.format("Message: %s", message)); + fixedAuxiliaryPool.submit(() -> { + try { + handle(message, schedulerDriver); + } catch (Exception e) { + log.warn("handle message error, message : " + message + ", cause : " + e.getMessage()); + try { + Thread.sleep(1000L); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + } + }); + } catch (CacheException e) { + log.warn("cache exception : " + e); + try { + Thread.sleep(10 * 1000L); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + } + } + } + + public static void kill(@NotNull Map killmap, String agentId, @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull FrameworkID frameworkID, @NotNull String url, final @NotNull long taskId) { + if (killmap.containsKey(taskId)) { + fixedAuxiliaryPool.submit(() -> { + JuiceTask juiceTask = daoUtils.queryTask(taskId); + if (null != juiceTask) { + if (juiceTask.getTaskStatus() > TaskResult.Result.RUNNING.getType()) { + killmap.remove(taskId); + return; + } + String value = killmap.get(taskId); + if (StringUtils.isBlank(value)) { + if (StringUtils.isNotBlank(juiceTask.getAgentId())) { + value = juiceTask.getAgentId(); + } else if (StringUtils.isNotBlank(agentId)) { + value = agentId; + } + } + if (StringUtils.isNotBlank(value)) { + TaskManagement.TaskAgentRel taskAgentRel = new TaskManagement.TaskAgentRel(taskId, juiceTask.getTaskName(), value); + Protos.Call call = SchedulerCalls.kill(frameworkID, taskAgentRel); + try { + SendUtils.sendCall(call, protocol, streamId, url); + killmap.remove(taskId); + } catch (IOException e) { + log.warn("send kill task call error due to : " + e.getCause()); + } + } + } + }); + } + } + + private static void handle(String message, SchedulerDriver schedulerDriver) throws IOException { + TaskManagement taskManagement = gson.fromJson(message, TaskManagement.class); + Protos.Call call = null; + if (null != taskManagement) { + switch (taskManagement.getType()) { + case KILL: + TaskManagement.TaskAgentRel taskAgentRel = taskManagement.getTaskAgentRels().get(0); + if (null == taskAgentRel) { + log.warn("object taskAgentRel is null"); + throw new DriverException("object taskAgentRel is null"); + } + JuiceTask juiceTask = daoUtils.queryTask(taskAgentRel.getTaskId()); + if (null != juiceTask) { + if (juiceTask.getTaskStatus() < TaskResult.Result.FINISHED.getType()) { + if (StringUtils.isBlank(juiceTask.getAgentId())) { + returnManagement(schedulerDriver, taskManagement, juiceTask); + return; + } + + taskAgentRel.setAgentId(juiceTask.getAgentId()); + call = SchedulerCalls.kill(schedulerDriver.getFrameworkId(), taskAgentRel); + try { + SendUtils.sendCall(call, schedulerDriver.getProtocol(), schedulerDriver.getStreamId(), schedulerDriver.getUrl()); + + } catch (IOException e) { + returnManagement(schedulerDriver, taskManagement, juiceTask); + return; + } + + if (schedulerDriver.getKillMap().containsKey(juiceTask.getTaskId())) { + schedulerDriver.getKillMap().remove(juiceTask.getTaskId()); + } + } + } + break; + case RECONCILE: + List taskAgentRels = taskManagement.getTaskAgentRels(); + if (null == taskAgentRels || taskAgentRels.size() == 0) { + log.warn("object taskAgentRel is null"); + throw new DriverException("object taskAgentRel is null"); + } + call = SchedulerCalls.reconcile(schedulerDriver.getFrameworkId(), taskAgentRels); + try { + SendUtils.sendCall(call, schedulerDriver.getProtocol(), schedulerDriver.getStreamId(), schedulerDriver.getUrl()); + } catch (IOException e) { + cacheUtils.pushToQueue(MANAGEMENT_QUEUE, message); + } + break; + default: + log.warn("unsupported type, type is : " + taskManagement.getType()); + throw new DriverException("unsupported type, type is : " + taskManagement.getType()); + } + return; + } + log.warn("object taskManagement is null"); + throw new DriverException("object taskManagement is null"); + } + private static void returnManagement(SchedulerDriver schedulerDriver, TaskManagement taskManagement, JuiceTask juiceTask) { + + // to reach TASK_RETRY_EXPIRE_TIME in seconds + if(taskManagement.getRetries() + TASK_RETRY_EXPIRE_TIME > currentTimeSeconds()) { + if (!schedulerDriver.getKillMap().containsKey(juiceTask.getTaskId())) { + schedulerDriver.getKillMap().put(juiceTask.getTaskId(), ""); + } + cacheUtils.pushToQueue(MANAGEMENT_QUEUE, gson.toJson(taskManagement)); + } + } + + public static void scheduledFramework(SchedulerDriver schedulerDriver) { + while (true) { + try { + // cached gen framework every hour + Thread.sleep(3600 * 1000L); + } catch (InterruptedException e) { + e.printStackTrace(); + } + String frameworkId = schedulerDriver.getFrameworkId().getValue(); + log.debug("framework_id --> " + frameworkId); + daoUtils.saveFrameworkId(MESOS_FRAMEWORK_TAG, frameworkId); + } + } + + public static void handleTaskErrors() { + while (true) { + try { + TaskResult taskResult = getTaskErrors().take(); + if (null != taskResult) { + try { + cacheUtils.pushToQueue(TASK_RESULT_QUEUE, gson.toJson(taskResult)); + } catch (CacheException e) { + log.error("redis cache is not available, write error to database"); + try { + daoUtils.finishTask(taskResult.getTaskId(), taskResult.getResult().getType(), taskResult.getMessage()); + } catch (DataBaseException e1) { + getTaskErrors().push(taskResult); + log.error("redis cache and db all error, write after 30's!"); + try { + Thread.sleep(30 * 1000L); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public static void handleSendErrors(SchedulerDriver schedulerDriver) { + while (true) { + try { + SendErrors errors = getSendErrors().take(); + if (schedulerDriver.getFrameworkId().getValue().equals(errors.getFrameworkId().getValue())) { + decline(schedulerDriver.getProtocol(), schedulerDriver.getStreamId(), errors.getFrameworkId(), errors.getCall(), schedulerDriver.getUrl()); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public static void decline(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull org.apache.mesos.v1.Protos.FrameworkID frameworkId, + final @NotNull Protos.Call call, final @NotNull String url) { + try { + SendUtils.sendCall(call, protocol, streamId, url); + } catch (IOException e) { + log.error("send decline call to mesos error!"); + log.error("frameworkId : " + frameworkId.getValue()); + log.error("call : " + call); + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + getSendErrors().push(new SendErrors(frameworkId, call)); + } + } + + public static void accept(final @NotNull Protocol protocol, final @NotNull String streamId, final @NotNull org.apache.mesos.v1.Protos.OfferID offerID, + final @NotNull org.apache.mesos.v1.Protos.FrameworkID frameworkID, final @NotNull List taskInfos, + final @NotNull String url) { + Protos.Call call = SchedulerCalls.accept(frameworkID, offerID, taskInfos); + log.info("accept --> Launching {} tasks", taskInfos.size()); + + try { + SendUtils.sendCall(call, protocol, streamId, url); + } catch (IOException e) { + log.error("send accept call to mesos error!"); + log.error("frameworkId : " + frameworkID.getValue() + ", offerId : " + offerID.getValue()); + log.error("call : " + call); + taskInfos.forEach( + taskInfo -> { + getTaskErrors().add(new TaskResult(com.hujiang.juice.common.model.Task.splitTaskNameId(taskInfo.getTaskId().getValue()), TaskResult.Result.ERROR, "send task to mesos error!")); + } + ); + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + List offerIDS = Lists.newArrayList(); + offerIDS.add(offerID); + + // if accept error, then offer will be decline + getSendErrors().push(new SendErrors(frameworkID, SchedulerCalls.decline(frameworkID, offerIDS))); + } + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/service/SchedulerService.java b/juice-service/src/main/java/com/hujiang/juice/service/service/SchedulerService.java new file mode 100644 index 0000000..dc87c29 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/service/SchedulerService.java @@ -0,0 +1,344 @@ +package com.hujiang.juice.service.service; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.hujiang.jooq.juice.tables.pojos.JuiceFramework; +import com.hujiang.juice.common.exception.CacheException; +import com.hujiang.juice.common.exception.DataBaseException; +import com.hujiang.juice.common.model.*; +import com.hujiang.juice.common.model.Task; +import com.hujiang.juice.service.utils.ResourcesUtils; +import com.hujiang.juice.common.vo.TaskResult; +import com.hujiang.juice.service.exception.UnrecoverException; +import com.hujiang.juice.service.support.Support; +import com.hujiang.juice.service.utils.protocol.Protocol; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.mesos.v1.Protos; +import org.jetbrains.annotations.NotNull; + +import java.util.*; + +import static com.hujiang.juice.common.config.COMMON.CPUS; +import static com.hujiang.juice.common.config.COMMON.MEMS; +import static com.hujiang.juice.common.utils.CommonUtils.currentTimeSeconds; +import static com.hujiang.juice.service.config.JUICE.*; +import static java.util.stream.Collectors.groupingBy; +import static org.apache.mesos.v1.Protos.*; +import static org.apache.mesos.v1.scheduler.Protos.Event; +import static org.apache.mesos.v1.scheduler.Protos.Event.Subscribed; + + +/** + * Created by xujia on 16/11/22. + */ +@Slf4j +public class SchedulerService { + + public static void removeFrameworkId() { + daoUtils.unActiveFramework(MESOS_FRAMEWORK_TAG); + } + + public static FrameworkID getFrameworkId() { + JuiceFramework frameworkInfo = null; + try { + frameworkInfo = daoUtils.queryFramework(MESOS_FRAMEWORK_TAG); + } catch (DataBaseException ex) { + log.error("get frameworkId from db failed {cause : db operation error}!"); + throw ex; + } + + if (null == frameworkInfo || StringUtils.isBlank(frameworkInfo.getFrameworkId())) { + return genFrameworkId(); + } else { + log.info("get framework id from db, id : " + frameworkInfo.getFrameworkId()); + return Protos.FrameworkID.newBuilder().setValue(frameworkInfo.getFrameworkId()).build(); + } + } + + public static FrameworkID genFrameworkId() { + String frameworkId = MESOS_SCHEDULER_NAME + "-" + UUID.randomUUID(); + log.info("generator a new framework id : " + frameworkId); + return Protos.FrameworkID.newBuilder().setValue(frameworkId).build(); + } + + public static void subscribed(Subscribed subscribed, FrameworkID frameworkID) { + if (null != subscribed) { + try { + if (frameworkID != null) { + daoUtils.saveFrameworkId(MESOS_FRAMEWORK_TAG, frameworkID.getValue()); + } + } catch (DataBaseException ex) { + log.error("save frameworkId error due to db operation error, frameworkId : " + frameworkID); + } + log.info("[subscribed success] id: " + subscribed.getFrameworkId() + ", heartbeat : " + subscribed.getHeartbeatIntervalSeconds()); + } + } + + public static boolean filterAndAddAttrSys(Offer offer, Map> attrMaps) { + if (!attrMaps.containsKey(offer.getId().getValue())) { + attrMaps.put(offer.getId().getValue(), Sets.newConcurrentHashSet()); + } + Set stringSet = attrMaps.get(offer.getId().getValue()); + BitSet bs = new BitSet(); + offer.getAttributesList().forEach( + ar -> { + Integer v = MESOS_FRAMEWORK_ATTR_FILTER_MAP.get(ar.getText().getValue()); + if (null != v && v >= 0) { + bs.set(v); + } + stringSet.add(ar.getText().getValue()); + } + ); + return bs.equals(MESOS_FRAMEWORK_ATTR_FILTER_DEFAULT_BITSET); + } + + public static void handleOffers(Map killMap, Support support, Offer offer, Set attributes, List declines, List tasks) { + final String desiredRole = support.getResourceRole(); + final AgentID agentId = offer.getAgentId(); + final OfferID offerId = offer.getId(); + + Map> facts = generatorFacts(offer, attributes); + + final Map> resources = offer.getResourcesList() + .stream() + .collect(groupingBy(Resource::getName)); + + final List cpuList = resources.get(CPUS); + final List memList = resources.get(MEMS); + + if (null != cpuList && !cpuList.isEmpty() + && null != memList && !memList.isEmpty() + && cpuList.size() == memList.size()) { + + for (int i = 0; i < cpuList.size(); i++) { + final Resource cpus = cpuList.get(i); + final Resource mem = memList.get(i); + + boolean isExhausted = false; + if (desiredRole.equals(cpus.getRole()) && desiredRole.equals(mem.getRole())) { + ResourcesUtils resourcesUtils = new ResourcesUtils(cpus.getScalar().getValue(), mem.getScalar().getValue(), RESOURCES_USE_THRESHOLD, desiredRole); + isExhausted = allocatingUntilExhausted(killMap, agentId, facts, resourcesUtils, tasks); + } + if (isExhausted) { + break; + } + } + + if(tasks.isEmpty()) { + declines.add(offerId); + } + } else { + declines.add(offerId); + } + } + + public static void update(Map killmap, TaskStatus taskStatus, @NotNull Protocol protocol, @NotNull FrameworkID frameworkID, @NotNull String streamId, @NotNull String url) { + long taskId = com.hujiang.juice.common.model.Task.splitTaskNameId(taskStatus.getTaskId().getValue()); + String message = taskStatus.getMessage(); + TaskResult.Result result = null; + log.debug("update --> taskId : " + taskId + ", task status : " + taskStatus.getState()); + switch (taskStatus.getState()) { + case TASK_STAGING: + message = StringUtils.isBlank(message) ? "task staging" : message; + result = TaskResult.Result.STAGING; + AuxiliaryService.kill(killmap, taskStatus.getAgentId().getValue(), protocol, streamId, frameworkID, url, taskId); + break; + case TASK_RUNNING: + message = StringUtils.isBlank(message) ? "task running" : message; + result = TaskResult.Result.RUNNING; + AuxiliaryService.kill(killmap, taskStatus.getAgentId().getValue(), protocol, streamId, frameworkID, url, taskId); + break; + case TASK_FINISHED: + message = StringUtils.isBlank(message) ? "task finished" : message; + result = TaskResult.Result.FINISHED; + break; + case TASK_FAILED: + message = StringUtils.isBlank(message) ? "task failed" : message; + result = TaskResult.Result.FAILED; + break; + case TASK_LOST: + message = StringUtils.isBlank(message) ? "task lost" : message; + result = TaskResult.Result.LOST; + break; + case TASK_ERROR: + message = StringUtils.isBlank(message) ? "task error" : message; + result = TaskResult.Result.ERROR; + break; + case TASK_KILLED: + message = StringUtils.isBlank(message) ? "task killed" : message; + result = TaskResult.Result.KILLED; + if(killmap.containsKey(taskId)) { + killmap.remove(taskId); + } + break; + case TASK_UNREACHABLE: + message = StringUtils.isBlank(message) ? "task unreachable" : message; + result = TaskResult.Result.UNREACHABLE; + break; + case TASK_DROPPED: + message = StringUtils.isBlank(message) ? "task dropped" : message; + result = TaskResult.Result.DROPPED; + break; + case TASK_GONE: + message = StringUtils.isBlank(message) ? "task gone" : message; + result = TaskResult.Result.GONE; + break; + case TASK_GONE_BY_OPERATOR: + message = StringUtils.isBlank(message) ? "task gone by operator" : message; + result = TaskResult.Result.GONE_BY_OPERATOR; + break; + case TASK_UNKNOWN: + message = StringUtils.isBlank(message) ? "task gone by unknown" : message; + result = TaskResult.Result.UNKNOWN; + break; + } + if (null != result) { + TaskResult taskResult = new TaskResult(taskId, result, message); + try { + cacheUtils.pushToQueue(TASK_RESULT_QUEUE, gson.toJson(taskResult)); + log.debug("update --> task result, taskResult [taskId : " + + taskResult.getTaskId() + + " status : " + + taskResult.getResult().name() + + " ]"); + } catch (CacheException e) { + AuxiliaryService.getTaskErrors().push(taskResult); + log.error("cache not available, push task result error, { taskId : " + taskId + " }"); + } + } + } + + public static void error(Event event) { + log.warn("event error --> " + event); + + String message = event.getError().getMessage(); + log.error("event error --> " + message); + if (message.toLowerCase().contains("framework has been removed") || message.toLowerCase().contains("framework is not authorized")) { + throw new UnrecoverException(message, true); + } else { + throw new UnrecoverException(message, false); + } + } + + public static void message(AgentID agentID, byte[] data) { + log.info("message, agentID : " + agentID + ", data : " + data); + } + + private static Map> generatorFacts(@NotNull Protos.Offer offer, Set attributes) { + Map> facts = Maps.newHashMap(); + // add host + if (StringUtils.isNotBlank(offer.getHostname())) { + facts.put(Constraints.FIELD.HOSTNAME.getField(), Sets.newHashSet()); + facts.get(Constraints.FIELD.HOSTNAME.getField()).add(offer.getHostname()); + } + // add attr + if (null != attributes && !attributes.isEmpty()) { + facts.put(Constraints.FIELD.RACK_ID.getField(), Sets.newHashSet()); + facts.get(Constraints.FIELD.RACK_ID.getField()).addAll(attributes); + } + + return facts; + } + + private static boolean allocatingUntilExhausted(Map killMap, Protos.AgentID agentId, Map> facts, ResourcesUtils hardware, List tasks) { + + long cacheTries = CACHE_TRIES; + // when either cpu or memory reach the picket line, will stop allocation task + while (hardware.isAvailable()) { + if (cacheTries > 0) { + cacheTries = taskOrResourceExhausted(killMap, agentId, facts, hardware, tasks, true) ? 0 : cacheTries - 1; + } else { + if (taskOrResourceExhausted(killMap, agentId, facts, hardware, tasks, false)) { + return true; + } + } + } + return false; + } + + private static boolean taskOrResourceExhausted(@NotNull Map killMap, Protos.AgentID agentId, Map> facts, ResourcesUtils hardware, List tasks, boolean isRetryTask) { + + // is task in cache exhausted ? + String tskStr = isRetryTask ? cacheUtils.popFromQueue(TASK_RETRY_QUEUE) : cacheUtils.popFromQueue(TASK_QUEUE); + if (StringUtils.isBlank(tskStr)) { + return true; + } + + // is resource exhausted ? + com.hujiang.juice.common.model.Task task = gson.fromJson(tskStr, Task.class); + + // current offer not match task, try next + if (!availableConstraints(task.getConstraints(), facts)) { + task.getExpire().incrementOfferLack(); + taskReserve(task, isRetryTask); + } else { + if (hardware.allocating(task.getResources())) { + // accept task + addTask(killMap, agentId, task, tasks, isRetryTask); + } else { + // resources is exhausted + task.getExpire().incrementResourceLack(); + taskReserve(task, isRetryTask); + return true; + } + } + return false; + + } + + private static boolean availableConstraints(Constraints constraints, Map> facts) { + if (null == constraints) { + return true; + } + return constraints.isAvailable(facts); + } + + private static void addTask(@NotNull Map killMap, @NotNull Protos.AgentID agentId, @NotNull Task task, @NotNull List tasks, boolean isRetryTask) { + boolean isToKilled = false; + try { + isToKilled = killMap.containsKey(task.getTaskId()); + //update db set taskAgentRel + AuxiliaryService.updateTask(killMap, task.getTaskId(), agentId.getValue(), isToKilled); + if(!isToKilled){ + tasks.add(task.getTask(agentId)); + log.info("resourceAllocation --> add task : " + task.getTaskId()); + } + } catch (Exception e) { + if(!isToKilled) { + taskReserve(task, isRetryTask); + } + } + } + + private static void taskReserve(Task task, boolean isRetryTask) { + if (isRetryTask) { + if (currentTimeSeconds() - task.getExpire().getFirstReservedTimes() > TASK_RETRY_EXPIRE_TIME) { + addToTaskErrors(task.getTaskId(), TaskResult.Result.EXPIRED, "task keep in queue's time > " + TASK_RETRY_EXPIRE_TIME + " seconds"); + } else if (task.getExpire().getOfferLack() > MAX_RESERVED) { + addToTaskErrors(task.getTaskId(), TaskResult.Result.ERROR, "lack of offer to handle task, constraints : " + task.getConstraints().toString()); + } else if (task.getExpire().getResourceLack() > MAX_RESERVED) { + addToTaskErrors(task.getTaskId(), TaskResult.Result.ERROR, "lack of resource to handle task, resources : " + task.getResources().toString()); + } else { + pushRetryTask(task); + } + } else { + task.getExpire().setFirstReservedTimes(currentTimeSeconds()); + pushRetryTask(task); + } + } + + private static void pushRetryTask(Task task) { + try { + cacheUtils.pushToQueue(TASK_RETRY_QUEUE, gson.toJson(task)); + } catch (CacheException e) { + log.error("cache exception, push task to retry-queue error, { taskId : " + task.getTaskId() + " }"); + addToTaskErrors(task.getTaskId(), TaskResult.Result.ERROR, "cache exception, push task to retry-queue error"); + throw e; + } + } + + private static void addToTaskErrors(long taskId, TaskResult.Result result, String message) { + AuxiliaryService.getTaskErrors().push(new TaskResult(taskId, result, message)); + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/support/Support.java b/juice-service/src/main/java/com/hujiang/juice/service/support/Support.java new file mode 100644 index 0000000..130f8d4 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/support/Support.java @@ -0,0 +1,22 @@ +package com.hujiang.juice.service.support; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +/** + * Created by xujia on 16/12/2. + */ + +@Slf4j +@Data +public class Support { + @NotNull + private final String resourceRole; + + public Support( + @NotNull final String resourceRole + ) { + this.resourceRole = resourceRole; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/LogInitUtil.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/LogInitUtil.java new file mode 100644 index 0000000..7f8b792 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/LogInitUtil.java @@ -0,0 +1,40 @@ +package com.hujiang.juice.service.utils; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import com.hujiang.juice.service.Startup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.LoggerFactory; + +import java.net.URL; + +/** + * Created by xujia on 17/2/7. + */ +@Slf4j +public class LogInitUtil { + + public static void initLog() { + LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator jc = new JoranConfigurator(); + jc.setContext(context); + context.reset(); + + String env = System.getProperty("system.environment"); + if(StringUtils.isBlank(env)) { + System.err.println("get system.environment error"); + throw new RuntimeException("can't get env, service stop!"); + } + URL tmpConfigFIleStr = Startup.class.getResource("/logback-" + env + ".xml"); + try { + System.out.println("start with configFile : " + tmpConfigFIleStr); + jc.doConfigure(tmpConfigFIleStr); + log.info("load logback config --> " + tmpConfigFIleStr.getFile()); + } catch (JoranException e) { + System.err.println(tmpConfigFIleStr + " not exist"); + throw new RuntimeException(e); + } + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/ResourcesUtils.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/ResourcesUtils.java new file mode 100644 index 0000000..186e24f --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/ResourcesUtils.java @@ -0,0 +1,36 @@ +package com.hujiang.juice.service.utils; + +import com.hujiang.juice.common.model.Resources; +import lombok.Data; +import org.jetbrains.annotations.NotNull; + +/** + * Created by xujia on 16/12/3. + */ + +@Data +public class ResourcesUtils { + + private int availableCpus; + private int availableMems; + private String role; + + public ResourcesUtils(double availableCpus, double availableMems, double threshold, String role) { + this.availableCpus = (int)(availableCpus * threshold); + this.availableMems = (int)(availableMems * threshold); + this.role = role; + } + + public boolean allocating(@NotNull Resources resources) { + if(resources.getCpu() <= availableCpus && resources.getMem() <= availableMems) { + availableCpus -= resources.getCpu(); + availableMems -= resources.getMem(); + return true; + } + return false; + } + + public boolean isAvailable(){ + return availableCpus >= 0 && availableMems >= 0; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/SendUtils.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/SendUtils.java new file mode 100644 index 0000000..25e6141 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/SendUtils.java @@ -0,0 +1,59 @@ +package com.hujiang.juice.service.utils; + +import com.google.protobuf.GeneratedMessage; +import com.hujiang.juice.common.utils.rest.Restty; +import com.hujiang.juice.service.utils.protocol.Protocol; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Created by xujia on 16/11/21. + */ +@Slf4j +public class SendUtils { + + public static void sendCall(GeneratedMessage call, Protocol protocol, String streamId, String url) throws IOException { + + log.debug("[call] " + call); + + Restty restty = Restty.create(url) + .addAccept(protocol.mediaType()) + .addMediaType(protocol.mediaType()) + .requestBody(protocol.getSendBytes(call)); + + if (StringUtils.isNotBlank(streamId)) { + restty.addHeader("Mesos-Stream-Id", streamId); + } + + try { + restty.postNoResponse(); + } catch (IOException e) { + log.warn("send call to mesos master failed, due to : " + e); + throw e; + } + + } + + + public static int readChunkSize(InputStream stream) throws IOException { + byte b; + + String s = ""; + while ((b = (byte) stream.read()) != '\n') + s += (char) b; + + return Integer.parseInt(s); + } + + public static byte[] readChunk(InputStream stream, int size) throws IOException { + byte[] buffer = new byte[size]; + + for (int i = 0; i < size; i++) + buffer[i] = (byte) stream.read(); + + return buffer; + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protobuf.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protobuf.java new file mode 100644 index 0000000..7bca7b2 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protobuf.java @@ -0,0 +1,34 @@ +package com.hujiang.juice.service.utils.protocol; + +import com.google.protobuf.GeneratedMessage; +import com.hujiang.juice.common.utils.rest.Restty; +import lombok.extern.slf4j.Slf4j; +import org.apache.mesos.v1.scheduler.Protos; + +import static org.apache.mesos.v1.executor.Protos.Event; + +/** + * Created by xujia on 16/11/28. + */ +@Slf4j +public class Protobuf extends Protocol { + + @Override + public byte[] getSendBytes(GeneratedMessage call) { + return call.toByteArray(); + } + + @Override + public String mediaType(){ + return Restty.protoBody(); + } + + @Override + public Object getEvent(byte[] buffer, Class classz) throws Exception { + if(classz.equals(Protos.Event.class)) { + return Protos.Event.parseFrom(buffer); + } + + return Event.parseFrom(buffer); + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protocol.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protocol.java new file mode 100644 index 0000000..dd582ef --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/protocol/Protocol.java @@ -0,0 +1,14 @@ +package com.hujiang.juice.service.utils.protocol; + +import com.google.protobuf.GeneratedMessage; + +/** + * Created by xujia on 16/11/28. + */ +public abstract class Protocol { + + public abstract byte[] getSendBytes(GeneratedMessage call); + public abstract String mediaType(); + + public abstract Object getEvent(byte[] buffer, Class classz) throws Exception; +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/CuratorUtils.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/CuratorUtils.java new file mode 100644 index 0000000..b7162d2 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/CuratorUtils.java @@ -0,0 +1,187 @@ +package com.hujiang.juice.service.utils.zookeeper; + +import com.hujiang.juice.common.utils.CommonUtils; +import com.hujiang.juice.service.exception.DriverException; +import com.hujiang.juice.service.model.Host; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.*; +import org.apache.curator.retry.ExponentialBackoffRetry; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.hujiang.juice.service.config.JUICE.*; + + +/** + * Created by xujia on 17/3/13. + */ + +@Slf4j +@Data +public class CuratorUtils { + + private Host host; + private volatile String path; + private CuratorFramework client; + private ExecutorService pool; + private NodeCache nodeCache; + + + public String generatePath() throws Exception { + return client.getChildren().forPath(MESOS_ROOT_PATH).stream().filter(ch -> ch.startsWith(PATH_NAME)).sorted().findFirst().map(this::getFullPath).orElse(null); + } + + private String getFullPath(String pathName) { + return MESOS_ROOT_PATH + HTTP_SEPERATOR + pathName; + } + + public CuratorUtils(String connectString, Host host) { + this.host = host; + + client = CuratorFrameworkFactory.builder() + .connectString(connectString) + .sessionTimeoutMs(10000) + .connectionTimeoutMs(10000) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + client.start(); + try { + path = generatePath(); + if (null == path) { + log.error("init curator failed due to path is null!"); + throw new DriverException("init curator failed due to path is null, servie will down"); + } + } catch (Exception e) { + log.error("CuratorUtils construct failed, service will down!"); + client.close(); + System.exit(-1); + } + } + + private void childrenCacheListenable() throws Exception { + final PathChildrenCache childrenCache = new PathChildrenCache(client, MESOS_ROOT_PATH, true); + childrenCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT); + childrenCache.getListenable().addListener( + new PathChildrenCacheListener() { + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) + throws Exception { + switch (event.getType()) { + case CHILD_ADDED: + added(); + break; + case CHILD_REMOVED: + removed(event.getData().getPath()); + break; + case CHILD_UPDATED: + updated(event.getData().getPath()); + break; + default: + break; + } + } + }, + pool + ); + } + + private void added() { + try { + if (StringUtils.isBlank(path)) { + String tmpPath = generatePath(); + if (StringUtils.isNotBlank(tmpPath)) { + path = tmpPath; + } + setHost(); + log.info("path changed to : " + path); + } + } catch (Exception e) { + log.error(e.getMessage()); + } + } + + private void removed(String changedPath) { + if (StringUtils.isNotBlank(changedPath) && changedPath.equals(path)) { + try { + path = generatePath(); + log.warn("path removed : " + path); + } catch (Exception e) { + path = null; + log.error(e.getMessage()); + } + } + } + + private void updated(String changedPath) { + try { + if(StringUtils.isBlank(path) || path.equals(changedPath)) { + if(StringUtils.isBlank(path)) { + String tmpPath = generatePath(); + if (StringUtils.isNotBlank(tmpPath)) { + path = tmpPath; + } + } + + setHost(); + } + } catch (Exception e) { + log.error(e.getMessage()); + } + + } + + private void setHost() { + try { + if (StringUtils.isNotBlank(path)) { + String connectString = getConnectString(); + if (StringUtils.isNotBlank(connectString) && (null == host.getHost() || !host.getHost().equals(connectString))) { + host.setHost(connectString); + log.info("path update, update mesos host to :" + host.getHost()); + } + } + } catch (Exception e) { + log.error(e.getMessage()); + } + } + + + private String getConnectString() throws Exception { + String connection = getLeaderConnectString(); + if (StringUtils.isNotBlank(connection)) { + log.info("connection -> " + connection); + return CommonUtils.fixUrl(connection); + } + return null; + } + + public String getLeaderConnectString() throws Exception { + String data = getData(path); + return gson.fromJson(data, JsonInfo.PathObject.class).getConnectString(); + } + + private String getData(String path) throws Exception { + return new String(client.getData().forPath(path)); + } + + public void init() { + try { + pool = Executors.newFixedThreadPool(1); + childrenCacheListenable(); + Thread.sleep(1000); + String connectString = getConnectString(); + if (StringUtils.isBlank(connectString)) { + throw new DriverException("get mesos host from zk error"); + } + host.setHost(connectString); + } catch (Exception e) { + String errMessage = "start curator listen failed, service will down!"; + log.error(errMessage); + pool.shutdown(); + System.exit(-1); + } + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/JsonInfo.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/JsonInfo.java new file mode 100644 index 0000000..ae13a21 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/JsonInfo.java @@ -0,0 +1,39 @@ +package com.hujiang.juice.service.utils.zookeeper; + +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +/** + * Created by xujia on 17/3/14. + */ + +@Data +public class JsonInfo { + @Data + public class PathObject { + Address address; + String hostname; + String id; + long ip; + String pid; + int port; + String version; + public String getConnectString() { + return address.getConnectString(); + } + } + + @Data + public class Address { + String hostname; + String ip; + int port; + + public String getConnectString() { + if(StringUtils.isNotBlank(ip) && port > 0) { + return ip + ":" + port; + } + return null; + } + } +} diff --git a/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/LeaderSelectorClient.java b/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/LeaderSelectorClient.java new file mode 100644 index 0000000..4b17ab7 --- /dev/null +++ b/juice-service/src/main/java/com/hujiang/juice/service/utils/zookeeper/LeaderSelectorClient.java @@ -0,0 +1,50 @@ +package com.hujiang.juice.service.utils.zookeeper; + +import com.hujiang.juice.service.driver.SchedulerDriver; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.leader.LeaderSelector; +import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.hujiang.juice.service.config.JUICE.*; + +/** + * Created by xujia on 17/3/31. + */ +@Slf4j +public class LeaderSelectorClient extends LeaderSelectorListenerAdapter implements Closeable { + private final LeaderSelector leaderSelector; + private final SchedulerDriver schedulerDriver; + private final AtomicInteger leaderCount = new AtomicInteger(); + + public LeaderSelectorClient(SchedulerDriver schedulerDriver) { + leaderSelector = new LeaderSelector(schedulerDriver.getCuratorUtils().getClient(), ZKLOCKS + HTTP_SEPERATOR + MESOS_FRAMEWORK_TAG, this); + leaderSelector.autoRequeue(); + this.schedulerDriver = schedulerDriver; + + } + + public void start() throws IOException { + leaderSelector.start(); + } + + @Override + public void takeLeadership(CuratorFramework client) throws Exception { + log.info("now the leader is " + (InetAddress.getLocalHost()).getHostName() + " , has been leader " + leaderCount.getAndIncrement() + " time(s) before."); + try { + schedulerDriver.run(); + } finally { + log.error("relinquishing leadership."); + } + } + + @Override + public void close() throws IOException { + leaderSelector.close(); + } +} diff --git a/juice-service/src/main/resources/application-dev.properties b/juice-service/src/main/resources/application-dev.properties new file mode 100644 index 0000000..6093308 --- /dev/null +++ b/juice-service/src/main/resources/application-dev.properties @@ -0,0 +1,18 @@ + +#for mesos boss start + +mesos.framework.zk= + +#for mesos attr filter +mesos.framework.attr= + +db.driver=com.mysql.jdbc.Driver +db.url= +db.user= +db.password= + + +#redis +redis.host= +redis.port= +redis.password= diff --git a/juice-service/src/main/resources/logback-dev.xml b/juice-service/src/main/resources/logback-dev.xml new file mode 100644 index 0000000..7c05f64 --- /dev/null +++ b/juice-service/src/main/resources/logback-dev.xml @@ -0,0 +1,56 @@ + + + + + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + + log/${APP_NAME}.log + + log/${APP_NAME}.%d{yyyy-MM-dd}.%i.log + + 365 + + + 20MB + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + + + log/ERROR.log + + log/ERROR.%d{yyyy-MM-dd}.%i.log + + 365 + + + 20MB + + + + [%date{ISO8601}][${APP_NAME}][${INSTANCE}][%5level][%logger:%L] %msg%n + + + ERROR + ACCEPT + DENY + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..8a7490c --- /dev/null +++ b/pom.xml @@ -0,0 +1,252 @@ + + + 4.0.0 + + com.hujiang + juice + pom + 1.0-OPEN + + + juice-service + juice-rest + juice-common + juice-jooq + juice-client-sdk + + + + a distribute tasks scheduling and allocating system based on mesos framework v1. + + + + hujiang Ltd + http://www.hujiang.com/ + + + + + xujia + xujia@hujiang.com + hujiang Ltd + China/Shanghai + + + + + UTF-8 + 1.8 + 1.1.0 + RELEASE + 1.10 + 3.8.2 + 1.7.21 + 1.1.7 + 1.16.10 + 5.1.39 + 1.0-OPEN + 1.0.20 + 2.12.0 + 3.3.1 + 2.9.0 + 3.4 + 2.5 + 20.0 + 2.7 + 3.1.0 + 2.7.4 + 0.6.12 + + + + + + + org.apache.curator + curator-recipes + ${curator.version} + + + com.hujiang + juice-common + ${juice.version} + + + com.hujiang + juice-auth + ${juice.version} + + + com.hujiang + juice-jooq + ${juice.version} + + + com.hujiang + juice-rest + ${juice.version} + + + com.hujiang + juice-service + ${juice.version} + + + com.hujiang + juice-test + ${juice.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + ch.qos.logback + logback-core + ${logback.version} + compile + + + ch.qos.logback + logback-classic + ${logback.version} + + + org.projectlombok + lombok + ${lombok.version} + + + commons-io + commons-io + ${commons-io.version} + + + com.google.guava + guava + ${guava.version} + + + com.google.code.gson + gson + ${gson.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + com.alibaba + druid + ${druid.version} + + + org.apache.mesos + mesos + ${mesos.version} + + + org.jetbrains + annotations-java5 + ${annotations.java5.version} + + + commons-codec + commons-codec + ${commons-codec.version} + + + org.jooq + jooq + ${jooq.version} + + + org.jooq + jooq-meta + ${jooq.version} + + + org.jooq + jooq-codegen + ${jooq.version} + + + mysql + mysql-connector-java + ${mysql.version} + + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + redis.clients + jedis + ${redis.clients.version} + + + javax.servlet + javax.servlet-api + ${servlet-api.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + org.msgpack + msgpack + ${msgpack.version} + + + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots/ + + true + + + + + true + + public + Public Repositories + http://nexus.yeshj.com/nexus/content/groups/public/ + + + + + + nexus + nexus + http://nexus.yeshj.com/nexus/content/groups/public/ + + true + + + true + + + + + + + releases + http://nexus.yeshj.com/nexus/content/repositories/releases + + + snapshots + http://nexus.yeshj.com/nexus/content/repositories/snapshots + + + \ No newline at end of file diff --git a/script/juice_2017-V1.0-OPEN.sql b/script/juice_2017-V1.0-OPEN.sql new file mode 100644 index 0000000..7d57421 --- /dev/null +++ b/script/juice_2017-V1.0-OPEN.sql @@ -0,0 +1,81 @@ +# ************************************************************ +# Sequel Pro SQL dump +# Version 4541 +# +# http://www.sequelpro.com/ +# https://github.com/sequelpro/sequelpro +# +# Host: 192.168.36.202 (MySQL 5.7.16) +# Database: juice +# Generation Time: 2017-03-28 04:40:36 +0000 +# ************************************************************ + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + + +# Dump of table juice_framework +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `juice_framework`; + +CREATE TABLE `juice_framework` ( + `framework_tag` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `framework_id` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `create_at` datetime DEFAULT CURRENT_TIMESTAMP, + `last_update_at` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + `is_active` int(1) NOT NULL DEFAULT '1', + PRIMARY KEY (`framework_tag`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + + + +# Dump of table juice_task +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `juice_task`; + +CREATE TABLE `juice_task` ( + `task_id` bigint(15) NOT NULL COMMENT '任务Id', + `task_name` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '服务名称', + `tenant_id` varchar(128) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '部门ID', + `docker_image` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT '' COMMENT 'docker镜像名', + `commands` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'commands', + `task_status` tinyint(1) NOT NULL COMMENT '任务状态', + `message` text COLLATE utf8mb4_unicode_ci COMMENT '任务说明', + `callback_at` datetime DEFAULT NULL COMMENT '回调时间', + `callback_url` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '回调URL', + `submit_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '任务提交时间', + `finish_at` datetime DEFAULT NULL, + `agent_id` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'agent id', + PRIMARY KEY (`task_id`,`submit_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci +/*!50100 PARTITION BY RANGE (month(submit_at)) +(PARTITION part0 VALUES LESS THAN (2) ENGINE = InnoDB, + PARTITION part1 VALUES LESS THAN (3) ENGINE = InnoDB, + PARTITION part2 VALUES LESS THAN (4) ENGINE = InnoDB, + PARTITION part3 VALUES LESS THAN (5) ENGINE = InnoDB, + PARTITION part4 VALUES LESS THAN (6) ENGINE = InnoDB, + PARTITION part5 VALUES LESS THAN (7) ENGINE = InnoDB, + PARTITION part6 VALUES LESS THAN (8) ENGINE = InnoDB, + PARTITION part7 VALUES LESS THAN (9) ENGINE = InnoDB, + PARTITION part8 VALUES LESS THAN (10) ENGINE = InnoDB, + PARTITION part9 VALUES LESS THAN (11) ENGINE = InnoDB, + PARTITION part10 VALUES LESS THAN (12) ENGINE = InnoDB, + PARTITION part11 VALUES LESS THAN (13) ENGINE = InnoDB) */; + + + + +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;