diff --git a/pom.xml b/pom.xml index 63f4c37..c2e6c33 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,12 @@ commons-httpclient commons-httpclient 3.0.1 + + + commons-codec + commons-codec + + com.google.code.gson @@ -82,6 +88,12 @@ org.apache.httpcomponents httpclient 4.5.2 + + + commons-codec + commons-codec + + com.google.guava diff --git a/src/main/java/com/gs/spider/controller/commons/spider/CommonsSpiderController.java b/src/main/java/com/gs/spider/controller/commons/spider/CommonsSpiderController.java index a30be3d..b6ec12f 100755 --- a/src/main/java/com/gs/spider/controller/commons/spider/CommonsSpiderController.java +++ b/src/main/java/com/gs/spider/controller/commons/spider/CommonsSpiderController.java @@ -8,6 +8,7 @@ import com.gs.spider.service.commons.spider.CommonsSpiderService; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.assertj.core.util.Lists; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Controller; @@ -139,4 +140,16 @@ public ResultListBundle getIgnoredUrls() { public ResultBundle addIgnoredUrl(String postfix) { return spiderService.addIgnoredUrl(postfix); } + + /** + * 根据爬虫模板ID批量启动任务 + * + * @param spiderInfoIdList 爬虫模板ID列表 + * @return 任务id列表 + */ + @RequestMapping(value = "startAll", method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public ResultListBundle startAll(String spiderInfoIdList) { + return spiderService.startAll(Lists.newArrayList(spiderInfoIdList.split(","))); + } } diff --git a/src/main/java/com/gs/spider/service/commons/spider/CommonsSpiderService.java b/src/main/java/com/gs/spider/service/commons/spider/CommonsSpiderService.java index 9abdf2b..e9bcd7d 100755 --- a/src/main/java/com/gs/spider/service/commons/spider/CommonsSpiderService.java +++ b/src/main/java/com/gs/spider/service/commons/spider/CommonsSpiderService.java @@ -1,6 +1,7 @@ package com.gs.spider.service.commons.spider; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.gson.Gson; import com.gs.spider.gather.async.AsyncGather; import com.gs.spider.gather.commons.CommonSpider; @@ -18,6 +19,8 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import javax.management.JMException; +import java.util.List; import java.util.Map; /** @@ -187,4 +190,26 @@ private void validateSpiderInfo(SpiderInfo spiderInfo) { "动态字段配置含有无效配置,每一个动态字段都必须有name,而且正则和xpath不可同时为空,请检查"); } } + + /** + * 根据爬虫模板ID批量启动任务 + * + * @param spiderInfoIdList 爬虫模板ID列表 + * @return 任务id列表 + */ + public ResultListBundle startAll(List spiderInfoIdList) { + return bundleBuilder.listBundle(spiderInfoIdList.toString(), () -> { + List taskIdList = Lists.newArrayList(); + for (String id : spiderInfoIdList) { + try { + SpiderInfo info = spiderInfoService.getById(id).getResult(); + String taskId = commonSpider.start(info); + taskIdList.add(taskId); + } catch (JMException e) { + LOG.error("启动任务ID{}出错,{}", id, e); + } + } + return taskIdList; + }); + } } diff --git a/src/main/webapp/js/my.js b/src/main/webapp/js/my.js index 1f0ba13..7b48d15 100755 --- a/src/main/webapp/js/my.js +++ b/src/main/webapp/js/my.js @@ -42,7 +42,7 @@ function rpc(url, pram, callback) { $('#confirmModal').modal('hide'); needShowResultModel = true; }); - $("#confirmModal").on('hidden.bs.modal', function () { + $("#confirmModal").one('hidden.bs.modal', function () { if (needShowResultModel) { $.getJSON(url, pram, callback); } diff --git a/src/main/webapp/pages/panel/commons/listSpiderInfo.jsp b/src/main/webapp/pages/panel/commons/listSpiderInfo.jsp index 6617914..e5c7dbc 100755 --- a/src/main/webapp/pages/panel/commons/listSpiderInfo.jsp +++ b/src/main/webapp/pages/panel/commons/listSpiderInfo.jsp @@ -34,6 +34,18 @@ }); }); + function checkAll() { + $('input:checkbox').each(function () { + $(this).attr('checked', true); + }); + } + function startAll() { + var idList = []; + $("input:checkbox:checked").each(function () { + idList.push($(this).attr('data-infoid')); + }); + rpcAndShowData('${pageContext.request.contextPath}/commons/spider/startAll', {spiderInfoIdList: idList.join(',')}); + } @@ -53,6 +65,10 @@
+
+ + +
@@ -67,6 +83,9 @@ +
${index.count} ${info.domain} ${info.siteName}