diff --git a/pom.xml b/pom.xml
index d6307ed..b42bb7f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
com.xjx
ddt-crawler
- 0.0.3-SNAPSHOT
+ 0.0.5-SNAPSHOT
ddt-crawler
ddt-crawler
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/CacheDomain.java b/src/main/java/com/xjx/ddtcrawler/cache/CacheDomain.java
index 842b162..e7be5dc 100644
--- a/src/main/java/com/xjx/ddtcrawler/cache/CacheDomain.java
+++ b/src/main/java/com/xjx/ddtcrawler/cache/CacheDomain.java
@@ -4,6 +4,8 @@
import lombok.Data;
import lombok.NoArgsConstructor;
+import java.io.Serializable;
+
/**
* @author XieJiaxing
* @date 2021/8/7 17:36
@@ -11,7 +13,16 @@
@Data
@NoArgsConstructor
@AllArgsConstructor
-public class CacheDomain {
+public class CacheDomain implements Serializable {
+ private static final long serialVersionUID = -1617868566077296126L;
+
private Object object;
private Long expireAt;
+
+ public boolean isExpired() {
+ if (expireAt == null) {
+ return false;
+ }
+ return expireAt < System.currentTimeMillis();
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/CacheInterface.java b/src/main/java/com/xjx/ddtcrawler/cache/CacheInterface.java
new file mode 100644
index 0000000..871d85d
--- /dev/null
+++ b/src/main/java/com/xjx/ddtcrawler/cache/CacheInterface.java
@@ -0,0 +1,12 @@
+package com.xjx.ddtcrawler.cache;
+
+/**
+ * @author XieJiaxing
+ * @date 2021/8/21 17:37
+ */
+public interface CacheInterface {
+ /**
+ * 清理过期的
+ */
+ void clearExpired();
+}
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/CommonCache.java b/src/main/java/com/xjx/ddtcrawler/cache/CommonCache.java
index 8039d71..50492be 100644
--- a/src/main/java/com/xjx/ddtcrawler/cache/CommonCache.java
+++ b/src/main/java/com/xjx/ddtcrawler/cache/CommonCache.java
@@ -1,5 +1,6 @@
package com.xjx.ddtcrawler.cache;
+import org.apache.commons.collections4.MapUtils;
import org.springframework.stereotype.Component;
import java.util.Map;
@@ -10,10 +11,13 @@
* @date 2021/8/7 17:33
*/
@Component
-public class CommonCache {
+public class CommonCache implements CacheInterface {
private final Map CACHE_MAP = new ConcurrentHashMap<>();
public void setCache(String key, Object value, Long expireTime) {
+ if (value == null) {
+ return;
+ }
Long expireAt = null;
if (expireTime != null) {
expireAt = System.currentTimeMillis() + expireTime;
@@ -49,4 +53,13 @@ public T getCache(String key) {
return null;
}
}
+
+ @Override
+ public void clearExpired() {
+ if (MapUtils.isEmpty(CACHE_MAP)) {
+ return;
+ }
+
+ CACHE_MAP.values().removeIf(CacheDomain::isExpired);
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/WebUserCache.java b/src/main/java/com/xjx/ddtcrawler/cache/WebUserCache.java
index 65c21cc..f1af0b8 100644
--- a/src/main/java/com/xjx/ddtcrawler/cache/WebUserCache.java
+++ b/src/main/java/com/xjx/ddtcrawler/cache/WebUserCache.java
@@ -1,7 +1,7 @@
package com.xjx.ddtcrawler.cache;
import com.xjx.ddtcrawler.cookie.WebUser;
-import com.xjx.ddtcrawler.utils.EncryptedUtils;
+import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
@@ -13,7 +13,7 @@
* @date 2021/8/1 22:56
*/
@Component
-public class WebUserCache {
+public class WebUserCache implements CacheInterface {
private final static Map MAP = new ConcurrentHashMap<>();
public WebUser getById(String id) {
@@ -25,7 +25,7 @@ public WebUser getById(String id) {
return null;
}
- Long expireTimeAt = webUser.getExpireTimeAt();
+ Long expireTimeAt = webUser.getExpireAt();
if (expireTimeAt == null) {
return webUser;
} else {
@@ -49,7 +49,7 @@ public String saveUser(WebUser webUser) {
Long userId = webUser.getUserId();
String key = webUser.getKey();
- Long expireTimeAt = webUser.getExpireTimeAt();
+ Long expireTimeAt = webUser.getExpireAt();
if (userId == null || StringUtils.isBlank(key)) {
return null;
}
@@ -84,4 +84,13 @@ public void delete(String id) {
MAP.remove(id);
}
+
+ @Override
+ public void clearExpired() {
+ if (MapUtils.isEmpty(MAP)) {
+ return;
+ }
+
+ MAP.values().removeIf(CacheDomain::isExpired);
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/controller/AuctionController.java b/src/main/java/com/xjx/ddtcrawler/controller/AuctionController.java
index 45f96b5..0e11e8d 100644
--- a/src/main/java/com/xjx/ddtcrawler/controller/AuctionController.java
+++ b/src/main/java/com/xjx/ddtcrawler/controller/AuctionController.java
@@ -24,6 +24,7 @@ public class AuctionController {
@RequestMapping("/auction/getAuctionItems")
public Result getItems(@RequestParam(required = false, defaultValue = "1") Integer page,
@RequestParam(required = false, defaultValue = "-1") Long userId,
+ @RequestParam(required = false, defaultValue = "-1") Long buyerId,
@RequestParam(required = false, defaultValue = "") String itemName,
@RequestParam(required = false, defaultValue = "2") Integer order,
@RequestParam(required = false, defaultValue = "true") Boolean sort) throws MyException, InterruptedException {
@@ -41,11 +42,13 @@ public Result getItems(@RequestParam(required = false, defaultValue = "1") Integ
queryUrl.setWebUser(webUser);
queryUrl.setOrder(order);
queryUrl.setUserId(userId);
+ queryUrl.setBuyId(buyerId);
queryUrl.setSort(sort);
queryUrl.setPage(page);
queryUrl.setName(itemName.trim());
if (temporaryUser) {
queryUrl.setUserId(-1L);
+ queryUrl.setBuyId(-1L);
}
Result result = auctionLogic.getSingleResult(queryUrl);
@@ -71,7 +74,7 @@ public Result getAuctionPriceOder(String itemName,
queryUrl.setSort(sort);
queryUrl.setName(itemName.trim());
- Result result = auctionLogic.getResultsByBatchPages(queryUrl, QueryUrl.DEFAULT_PAGES, true, 250L, priceType, sort);
+ Result result = auctionLogic.getResultsByBatchPages(queryUrl, QueryUrl.DEFAULT_PAGES, true, 250L, priceType, sort, null);
if (webUser.isTemporaryUser()) {
result.hideSensitiveInfo();
}
diff --git a/src/main/java/com/xjx/ddtcrawler/controller/TemplateController.java b/src/main/java/com/xjx/ddtcrawler/controller/TemplateController.java
index 7ee5071..2b28b3f 100644
--- a/src/main/java/com/xjx/ddtcrawler/controller/TemplateController.java
+++ b/src/main/java/com/xjx/ddtcrawler/controller/TemplateController.java
@@ -8,6 +8,8 @@
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
+import java.util.List;
+
/**
* @author XieJiaxing
* @date 2021/8/14 18:33
@@ -26,7 +28,6 @@ public Template getTemplateById(Long id) throws MyException {
return templateLogic.getTemplateById(id);
}
-
@RequestMapping("/template/saveTemplate")
public Boolean saveTemplate(Long templateId, String templateName) throws MyException {
WebUser webUser = WebUser.getSafeWebUser();
@@ -35,4 +36,14 @@ public Boolean saveTemplate(Long templateId, String templateName) throws MyExcep
}
return templateLogic.saveTemplate(templateId, templateName);
}
+
+ @RequestMapping("/template/listAllTemplates")
+ public List listAllTemplates() {
+ return templateLogic.listAllTemplates();
+ }
+
+ @RequestMapping("/template/listCommonSlv4")
+ public List listCommonSlv4() {
+ return templateLogic.listCommonSlv4();
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/controller/UserController.java b/src/main/java/com/xjx/ddtcrawler/controller/UserController.java
index 40500ac..9f0fd80 100644
--- a/src/main/java/com/xjx/ddtcrawler/controller/UserController.java
+++ b/src/main/java/com/xjx/ddtcrawler/controller/UserController.java
@@ -51,13 +51,16 @@ public MyMessage createTemporaryLicense(Long expireTime, @RequestParam(required
}
long minConcurrentTime = 100L;
+ if (concurrentTime < webUser.getConcurrentTime()) {
+ concurrentTime = webUser.getConcurrentTime();
+ }
if (concurrentTime < minConcurrentTime) {
concurrentTime = minConcurrentTime;
}
long now = System.currentTimeMillis();
if (expireTime != null) {
- webUser.setExpireTimeAt(expireTime * 1000 + now);
+ webUser.setExpireAt(expireTime * 1000 + now);
}
webUser.setConcurrentTime(concurrentTime);
String temporaryLicense = webUserLogic.generateTemporaryLicense(webUser);
diff --git a/src/main/java/com/xjx/ddtcrawler/cookie/WebUser.java b/src/main/java/com/xjx/ddtcrawler/cookie/WebUser.java
index a4887f2..517035d 100644
--- a/src/main/java/com/xjx/ddtcrawler/cookie/WebUser.java
+++ b/src/main/java/com/xjx/ddtcrawler/cookie/WebUser.java
@@ -1,8 +1,10 @@
package com.xjx.ddtcrawler.cookie;
+import com.xjx.ddtcrawler.cache.CacheDomain;
import com.xjx.ddtcrawler.exception.MyException;
import com.xjx.ddtcrawler.utils.EncryptedUtils;
import lombok.Data;
+import lombok.EqualsAndHashCode;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
@@ -12,8 +14,9 @@
* @author XieJiaxing
* @date 2021/8/14 12:29
*/
+@EqualsAndHashCode(callSuper = true)
@Data
-public class WebUser implements Serializable {
+public class WebUser extends CacheDomain implements Serializable {
private static final long serialVersionUID = 5392034256168678992L;
public static final ThreadLocal WEB_USER_THREAD_LOCAL = new ThreadLocal<>();
@@ -22,12 +25,11 @@ public class WebUser implements Serializable {
private Long loginTime;
private Long userId;
private String key;
- private Long expireTimeAt;
private String temporaryLicense;
/**
* 用户并发毫秒
*/
- private Long concurrentTime = 1000L;
+ private Long concurrentTime = 500L;
public static void setWebUser(WebUser webUser) {
WEB_USER_THREAD_LOCAL.set(webUser);
@@ -42,7 +44,7 @@ public static WebUser getSafeWebUser() throws MyException {
if (webUser == null) {
throw new MyException("用户不存在或者已过期");
}
- Long expireTimeAt = webUser.getExpireTimeAt();
+ Long expireTimeAt = webUser.getExpireAt();
if (expireTimeAt != null && expireTimeAt < System.currentTimeMillis()) {
throw new MyException("用户不存在或者已过期");
}
diff --git a/src/main/java/com/xjx/ddtcrawler/domain/Item.java b/src/main/java/com/xjx/ddtcrawler/domain/Item.java
index 6083399..c9df587 100644
--- a/src/main/java/com/xjx/ddtcrawler/domain/Item.java
+++ b/src/main/java/com/xjx/ddtcrawler/domain/Item.java
@@ -3,6 +3,7 @@
import com.fasterxml.jackson.annotation.JsonInclude;
import com.thoughtworks.xstream.annotations.XStreamAlias;
import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+import com.xjx.ddtcrawler.utils.EncryptedUtils;
import com.xjx.ddtcrawler.utils.TimeUtil;
import lombok.Data;
@@ -162,6 +163,29 @@ public class Item implements Serializable {
* 用户定义的低价
*/
private Double userDefinePrice;
+ private Boolean isTemplateIdEncrypted = false;
+
+ public void encryptTemplateId() {
+ if (this.isTemplateIdEncrypted == null) {
+ this.isTemplateIdEncrypted = false;
+ }
+ if (this.isTemplateIdEncrypted) {
+ return;
+ }
+ this.templateId = EncryptedUtils.encryptTemplateId(this.templateId);
+ this.isTemplateIdEncrypted = true;
+ }
+
+ public void decryptTemplateId() {
+ if (this.isTemplateIdEncrypted == null) {
+ this.isTemplateIdEncrypted = false;
+ }
+ if (!this.isTemplateIdEncrypted) {
+ return;
+ }
+ this.templateId = EncryptedUtils.decryptTemplateId(this.templateId);
+ this.isTemplateIdEncrypted = false;
+ }
@Override
public String toString() {
diff --git a/src/main/java/com/xjx/ddtcrawler/domain/QueryUrl.java b/src/main/java/com/xjx/ddtcrawler/domain/QueryUrl.java
index b8ffc5f..9f21b40 100644
--- a/src/main/java/com/xjx/ddtcrawler/domain/QueryUrl.java
+++ b/src/main/java/com/xjx/ddtcrawler/domain/QueryUrl.java
@@ -98,7 +98,9 @@ public String buildUrl() throws MyException {
rnd = RANDOM.nextDouble();
String format = "?page=%s&selfid=%s&type=%s&Auctions=%s&buyID=%s&key=%s&name=%s&userId=%s&rnd=%s&pay=%s&order=%s&sort=%s";
- return URL + String.format(format, page, selfId, type, auctions, buyId, key, ec(name), userId, rnd, pay, order, sort);
+ String url = URL + String.format(format, page, selfId, type, auctions, buyId, key, ec(name), userId, rnd, pay, order, sort);
+ log.info(url);
+ return url;
}
/**
diff --git a/src/main/java/com/xjx/ddtcrawler/domain/Result.java b/src/main/java/com/xjx/ddtcrawler/domain/Result.java
index 62a15bf..632433f 100644
--- a/src/main/java/com/xjx/ddtcrawler/domain/Result.java
+++ b/src/main/java/com/xjx/ddtcrawler/domain/Result.java
@@ -95,7 +95,6 @@ public void hideSensitiveInfo() {
item.setBuyerName("有");
}
item.setAuctioneerId(null);
- item.setTemplateId(null);
item.setBeginDate(null);
item.setAuctionDate(null);
item.setItemDate(null);
@@ -104,6 +103,7 @@ public void hideSensitiveInfo() {
item.setUserDefinePrice(null);
Long beginTime = item.getBeginTime();
item.setBeginTime(null);
+ item.encryptTemplateId();
if (beginTime != null) {
long v = (System.currentTimeMillis() - beginTime) / 1000 / 3600;
diff --git a/src/main/java/com/xjx/ddtcrawler/domain/Template.java b/src/main/java/com/xjx/ddtcrawler/domain/Template.java
index 8b17b8f..dae0daf 100644
--- a/src/main/java/com/xjx/ddtcrawler/domain/Template.java
+++ b/src/main/java/com/xjx/ddtcrawler/domain/Template.java
@@ -1,5 +1,6 @@
package com.xjx.ddtcrawler.domain;
+import com.xjx.ddtcrawler.utils.EncryptedUtils;
import lombok.Data;
import java.io.Serializable;
@@ -16,4 +17,21 @@ public class Template implements Serializable {
private String name;
private Double price;
private Long modifyTime;
+ private Boolean isEncrypted = false;
+
+ public void encryptId() {
+ if (this.isEncrypted) {
+ return;
+ }
+ this.id = EncryptedUtils.encryptTemplateId(this.id);
+ this.isEncrypted = true;
+ }
+
+ public void decryptId() {
+ if (!this.isEncrypted) {
+ return;
+ }
+ this.id = EncryptedUtils.decryptTemplateId(this.id);
+ this.isEncrypted = false;
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/http/HttpHelper.java b/src/main/java/com/xjx/ddtcrawler/http/HttpHelper.java
index 075fe1f..968557f 100644
--- a/src/main/java/com/xjx/ddtcrawler/http/HttpHelper.java
+++ b/src/main/java/com/xjx/ddtcrawler/http/HttpHelper.java
@@ -2,6 +2,7 @@
import com.xjx.ddtcrawler.http.enumeration.MethodEnum;
import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.http.*;
import org.apache.http.client.CookieStore;
import org.apache.http.client.config.CookieSpecs;
@@ -10,6 +11,7 @@
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.utils.HttpClientUtils;
import org.apache.http.conn.params.ConnRouteParams;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
@@ -17,8 +19,12 @@
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
+import javax.net.ssl.*;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
import java.util.List;
/**
@@ -35,7 +41,31 @@ public class HttpHelper {
public HttpHelper(HttpRequestMaker httpRequestMaker) {
this.httpRequestMaker = httpRequestMaker;
httpCookieStore = new BasicCookieStore();
- this.httpClient = HttpClientBuilder.create().setDefaultRequestConfig(RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build()).setDefaultCookieStore(httpCookieStore).build();
+ X509TrustManager x509mgr = new X509TrustManager() {
+ @Override
+ public void checkClientTrusted(X509Certificate[] xcs, String string) {
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] xcs, String string) {
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ };
+ SSLConnectionSocketFactory sslsf = null;
+ try {
+ SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(null, new TrustManager[]{x509mgr}, null);
+ HostnameVerifier hostnameVerifier = (hostname, session) -> true;
+ sslsf = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
+ } catch (NoSuchAlgorithmException | KeyManagementException e) {
+ e.printStackTrace();
+ }
+
+ this.httpClient = HttpClientBuilder.create().setSSLSocketFactory(sslsf).setDefaultRequestConfig(RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build()).setDefaultCookieStore(httpCookieStore).build();
}
public HttpHelper(String url, MethodEnum methodEnum) {
diff --git a/src/main/java/com/xjx/ddtcrawler/logic/AuctionLogic.java b/src/main/java/com/xjx/ddtcrawler/logic/AuctionLogic.java
index a4d9ba3..31c5d44 100644
--- a/src/main/java/com/xjx/ddtcrawler/logic/AuctionLogic.java
+++ b/src/main/java/com/xjx/ddtcrawler/logic/AuctionLogic.java
@@ -75,20 +75,29 @@ public Result getRawResponseBody(QueryUrl queryUrl) throws MyException {
public Result getSingleResult(QueryUrl queryUrl) throws MyException, InterruptedException {
List pages = new ArrayList<>();
pages.add(queryUrl.getPage());
- return getResultsByBatchPages(queryUrl, pages, true, 250L, null, null);
+ Long userId = queryUrl.getUserId();
+ Long buyId = queryUrl.getBuyId();
+ if ((userId != null && userId > 0) || (buyId != null && buyId > 0)) {
+ pages = QueryUrl.DEFAULT_PAGES;
+ }
+ return getResultsByBatchPages(queryUrl, pages, true, 250L, null, null, null);
}
/**
* 批量获取多页的拍卖场物品信息
*
- * @param queryUrl queryUrl
- * @param pages 页码集合
- * @param sleep 时延，毫秒
+ * @param queryUrl queryUrl
+ * @param pages 页码集合
+ * @param withTemplate withTemplate
+ * @param sleep 延迟毫秒
+ * @param priceSortType 价格排序规则，竞拍单价、一口单价
+ * @param sortOrder 排序规则，{@link AuctionConstant.PriceSortTypeEnum}
+ * @param minAuctionId 最小的 auctionId，如果获取到的 result 里面的 auctionId 小于入参，那么结束循环
* @return Result
- * @throws MyException exception
- * @throws InterruptedException exception
+ * @throws MyException Exception
+ * @throws InterruptedException Exception
*/
- public Result getResultsByBatchPages(QueryUrl queryUrl, List pages, boolean withTemplate, Long sleep, Integer priceSortType, Boolean sortOrder) throws MyException, InterruptedException {
+ public Result getResultsByBatchPages(QueryUrl queryUrl, List pages, boolean withTemplate, Long sleep, Integer priceSortType, Boolean sortOrder, Long minAuctionId) throws MyException, InterruptedException {
if (CollectionUtils.isNotEmpty(pages)) {
pages.removeIf(p -> p <= 0);
// 去重
@@ -133,8 +142,10 @@ public Result getResultsByBatchPages(QueryUrl queryUrl, List pages, boo
i = pages.size();
}
+ List- items = result.getItems();
+
// 每次成功后 total 累加，items 累加
- totalResult.getItems().addAll(result.getItems());
+ totalResult.getItems().addAll(items);
if (i + 1 < pages.size()) {
// 每次请求之后的时延，防止高并发
diff --git a/src/main/java/com/xjx/ddtcrawler/logic/TemplateLogic.java b/src/main/java/com/xjx/ddtcrawler/logic/TemplateLogic.java
index 58195bf..f3024d3 100644
--- a/src/main/java/com/xjx/ddtcrawler/logic/TemplateLogic.java
+++ b/src/main/java/com/xjx/ddtcrawler/logic/TemplateLogic.java
@@ -3,16 +3,32 @@
import com.xjx.ddtcrawler.domain.Template;
import com.xjx.ddtcrawler.exception.MyException;
import com.xjx.ddtcrawler.service.TemplateService;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
/**
* @author XieJiaxing
* @date 2021/8/14 18:34
*/
@Component
public class TemplateLogic {
+ private static final List COMMON_SLV_4 = new ArrayList<>();
+
+ static {
+ Long[] commonAttributeSlv4 = {311418L, 311420L, 311421L, 311422L, 311423L, 311424L, 311426L, 311427L, 311428L, 311429L, 311430L, 311431L, 311432L, 311433L, 311434L};
+ Long[] commonAttackSlv4 = {312409L, 312410L, 312411L, 312412L, 312413L, 312415L};
+ Long[] commonDefenceSlv4 = {313407L, 313408L, 313409L, 313410L, 313412L};
+ COMMON_SLV_4.addAll(Arrays.asList(commonAttributeSlv4));
+ COMMON_SLV_4.addAll(Arrays.asList(commonAttackSlv4));
+ COMMON_SLV_4.addAll(Arrays.asList(commonDefenceSlv4));
+ }
+
@Autowired
private TemplateService templateService;
@@ -29,4 +45,20 @@ public Boolean saveTemplate(Long templateId, String templateName) throws MyExcep
template.setName(templateName);
return templateService.save(template);
}
+
+ public List listAllTemplates() {
+ List templates = templateService.getAll();
+ templates.forEach(Template::encryptId);
+ return templates;
+ }
+
+ public List listCommonSlv4() {
+ List templates = templateService.getByIds(COMMON_SLV_4);
+ if (CollectionUtils.isEmpty(templates)) {
+ return new ArrayList<>();
+ }
+
+ templates.forEach(Template::encryptId);
+ return templates;
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/logic/WebUserLogic.java b/src/main/java/com/xjx/ddtcrawler/logic/WebUserLogic.java
index 385cedf..feee92b 100644
--- a/src/main/java/com/xjx/ddtcrawler/logic/WebUserLogic.java
+++ b/src/main/java/com/xjx/ddtcrawler/logic/WebUserLogic.java
@@ -1,11 +1,9 @@
package com.xjx.ddtcrawler.logic;
-import com.xjx.ddtcrawler.cache.CommonCache;
import com.xjx.ddtcrawler.cache.WebUserCache;
import com.xjx.ddtcrawler.cookie.CookieHelper;
import com.xjx.ddtcrawler.cookie.WebUser;
import com.xjx.ddtcrawler.exception.MyException;
-import com.xjx.ddtcrawler.utils.EncryptedUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
@@ -43,7 +41,7 @@ public void save(Long userId, String key, Long expireTime) throws MyException {
int maxAge = CookieHelper.COOKIE_MAX_AGE;
if (expireTime != null) {
- webUser.setExpireTimeAt(expireTime + now);
+ webUser.setExpireAt(expireTime + now);
maxAge = (int) (expireTime / 1000);
}
String id = webUserCache.saveUser(webUser);
@@ -97,7 +95,7 @@ public void writeTemporaryLicenseCookie(String temporaryLicense) throws MyExcept
}
int maxAge = CookieHelper.COOKIE_MAX_AGE;
- Long expireTimeAt = webUser.getExpireTimeAt();
+ Long expireTimeAt = webUser.getExpireAt();
if (expireTimeAt != null) {
if (expireTimeAt < System.currentTimeMillis()) {
// webUser 已经过期，删除 cookie
diff --git a/src/main/java/com/xjx/ddtcrawler/mapper/TemplateMapper.java b/src/main/java/com/xjx/ddtcrawler/mapper/TemplateMapper.java
index 4463f3f..1904e99 100644
--- a/src/main/java/com/xjx/ddtcrawler/mapper/TemplateMapper.java
+++ b/src/main/java/com/xjx/ddtcrawler/mapper/TemplateMapper.java
@@ -15,6 +15,8 @@ public interface TemplateMapper {
List getByIds(List ids);
+ List getAll();
+
void add(Template template);
void update(Template template);
diff --git a/src/main/java/com/xjx/ddtcrawler/task/MySqlConnectionTask.java b/src/main/java/com/xjx/ddtcrawler/schedule/MySchedule.java
similarity index 59%
rename from src/main/java/com/xjx/ddtcrawler/task/MySqlConnectionTask.java
rename to src/main/java/com/xjx/ddtcrawler/schedule/MySchedule.java
index 97cab98..474fae5 100644
--- a/src/main/java/com/xjx/ddtcrawler/task/MySqlConnectionTask.java
+++ b/src/main/java/com/xjx/ddtcrawler/schedule/MySchedule.java
@@ -1,9 +1,11 @@
-package com.xjx.ddtcrawler.task;
+package com.xjx.ddtcrawler.schedule;
+import com.xjx.ddtcrawler.cache.CacheInterface;
import com.xjx.ddtcrawler.domain.Template;
import com.xjx.ddtcrawler.service.TemplateService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
@@ -12,18 +14,25 @@
/**
* @author XieJiaxing
- * @date 2021/8/7 14:45
+ * @date 2021/8/21 17:29
*/
@Configuration
@EnableScheduling
@Slf4j
-public class MySqlConnectionTask {
+public class MySchedule {
+ private static final Random RANDOM = new Random();
+
@Autowired
private TemplateService templateService;
- private static final Random RANDOM = new Random();
+ @Autowired
+ @Qualifier("commonCache")
+ private CacheInterface commonCache;
+ @Autowired
+ @Qualifier("webUserCache")
+ private CacheInterface webUserCache;
@Scheduled(cron = "0 */1 * * * ?")
- private void configureTasks() {
+ public void mySqlConnection() {
try {
Template template = templateService.getById(RANDOM.nextLong() / 1000);
log.info("test connection");
@@ -31,4 +40,11 @@ private void configureTasks() {
log.error("test connection error", e);
}
}
+
+ @Scheduled(cron = "0 */10 * * * ?")
+ public void clearExpireCache() {
+ commonCache.clearExpired();
+ webUserCache.clearExpired();
+ log.info("clearExpireCache");
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/service/TemplateService.java b/src/main/java/com/xjx/ddtcrawler/service/TemplateService.java
index 64bbfcb..1c469c7 100644
--- a/src/main/java/com/xjx/ddtcrawler/service/TemplateService.java
+++ b/src/main/java/com/xjx/ddtcrawler/service/TemplateService.java
@@ -67,4 +67,8 @@ public boolean save(Template template) {
return true;
}
+
+ public List getAll() {
+ return templateMapper.getAll();
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/utils/EncryptedUtils.java b/src/main/java/com/xjx/ddtcrawler/utils/EncryptedUtils.java
index 7881d5a..6c0edd7 100644
--- a/src/main/java/com/xjx/ddtcrawler/utils/EncryptedUtils.java
+++ b/src/main/java/com/xjx/ddtcrawler/utils/EncryptedUtils.java
@@ -48,4 +48,27 @@ public static Long decryptTemporaryLicense(String s) {
String substring = s.substring(1, s.length() - 1);
return Long.parseLong(substring) ^ KEY;
}
+
+ public static Long encryptTemplateId(Long id) {
+ String s = encryptUserId(id);
+ if (StringUtils.isBlank(s)) {
+ return null;
+ }
+
+ return Long.valueOf(s);
+ }
+
+ public static Long decryptTemplateId(Long id) {
+ if (id == null) {
+ return null;
+ }
+
+ String s = String.valueOf(id);
+ if (StringUtils.isBlank(s) || s.length() <= 2 || !StringUtils.isNumeric(s)) {
+ return null;
+ }
+
+ String substring = s.substring(1, s.length() - 1);
+ return Long.parseLong(substring) ^ KEY;
+ }
}
diff --git a/src/main/java/com/xjx/ddtcrawler/utils/InstallCert.java b/src/main/java/com/xjx/ddtcrawler/utils/InstallCert.java
new file mode 100644
index 0000000..a1cf48c
--- /dev/null
+++ b/src/main/java/com/xjx/ddtcrawler/utils/InstallCert.java
@@ -0,0 +1,190 @@
+package com.xjx.ddtcrawler.utils;
+/*
+ * Copyright 2006 Sun Microsystems, Inc. All Rights Reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of Sun Microsystems nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import javax.net.ssl.*;
+import java.io.*;
+import java.security.KeyStore;
+import java.security.MessageDigest;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+public class InstallCert {
+
+ public static void main(String[] args) throws Exception {
+ args = new String[]{"s57_app1105673153_qqgame_com.7road.net"};
+ String host;
+ int port;
+ char[] passphrase;
+ if ((args.length == 1) || (args.length == 2)) {
+ String[] c = args[0].split(":");
+ host = c[0];
+ port = (c.length == 1) ? 443 : Integer.parseInt(c[1]);
+ String p = (args.length == 1) ? "changeit" : args[1];
+ passphrase = p.toCharArray();
+ } else {
+ System.out.println("Usage: java InstallCert [:port] [passphrase]");
+ return;
+ }
+
+ File file = new File("jssecacerts");
+ if (file.isFile() == false) {
+ char SEP = File.separatorChar;
+ File dir = new File(System.getProperty("java.home") + SEP
+ + "lib" + SEP + "security");
+ file = new File(dir, "jssecacerts");
+ if (file.isFile() == false) {
+ file = new File(dir, "cacerts");
+ }
+ }
+ System.out.println("Loading KeyStore " + file + "...");
+ InputStream in = new FileInputStream(file);
+ KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
+ ks.load(in, passphrase);
+ in.close();
+
+ SSLContext context = SSLContext.getInstance("TLS");
+ TrustManagerFactory tmf =
+ TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(ks);
+ X509TrustManager defaultTrustManager = (X509TrustManager) tmf.getTrustManagers()[0];
+ SavingTrustManager tm = new SavingTrustManager(defaultTrustManager);
+ context.init(null, new TrustManager[]{tm}, null);
+ SSLSocketFactory factory = context.getSocketFactory();
+
+ System.out.println("Opening connection to " + host + ":" + port + "...");
+ SSLSocket socket = (SSLSocket) factory.createSocket(host, port);
+ socket.setSoTimeout(10000);
+ try {
+ System.out.println("Starting SSL handshake...");
+ socket.startHandshake();
+ socket.close();
+ System.out.println();
+ System.out.println("No errors, certificate is already trusted");
+ } catch (SSLException e) {
+ System.out.println();
+ e.printStackTrace(System.out);
+ }
+
+ X509Certificate[] chain = tm.chain;
+ if (chain == null) {
+ System.out.println("Could not obtain server certificate chain");
+ return;
+ }
+
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(System.in));
+
+ System.out.println();
+ System.out.println("Server sent " + chain.length + " certificate(s):");
+ System.out.println();
+ MessageDigest sha1 = MessageDigest.getInstance("SHA1");
+ MessageDigest md5 = MessageDigest.getInstance("MD5");
+ for (int i = 0; i < chain.length; i++) {
+ X509Certificate cert = chain[i];
+ System.out.println
+ (" " + (i + 1) + " Subject " + cert.getSubjectDN());
+ System.out.println(" Issuer " + cert.getIssuerDN());
+ sha1.update(cert.getEncoded());
+ System.out.println(" sha1 " + toHexString(sha1.digest()));
+ md5.update(cert.getEncoded());
+ System.out.println(" md5 " + toHexString(md5.digest()));
+ System.out.println();
+ }
+
+ System.out.println("Enter certificate to add to trusted keystore or 'q' to quit: [1]");
+ String line = reader.readLine().trim();
+ int k;
+ try {
+ k = (line.length() == 0) ? 0 : Integer.parseInt(line) - 1;
+ } catch (NumberFormatException e) {
+ System.out.println("KeyStore not changed");
+ return;
+ }
+
+ X509Certificate cert = chain[k];
+ String alias = host + "-" + (k + 1);
+ ks.setCertificateEntry(alias, cert);
+
+ OutputStream out = new FileOutputStream("jssecacerts");
+ ks.store(out, passphrase);
+ out.close();
+
+ System.out.println();
+ System.out.println(cert);
+ System.out.println();
+ System.out.println
+ ("Added certificate to keystore 'jssecacerts' using alias '"
+ + alias + "'");
+ }
+
+ private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray();
+
+ private static String toHexString(byte[] bytes) {
+ StringBuilder sb = new StringBuilder(bytes.length * 3);
+ for (int b : bytes) {
+ b &= 0xff;
+ sb.append(HEXDIGITS[b >> 4]);
+ sb.append(HEXDIGITS[b & 15]);
+ sb.append(' ');
+ }
+ return sb.toString();
+ }
+
+ private static class SavingTrustManager implements X509TrustManager {
+
+ private final X509TrustManager tm;
+ private X509Certificate[] chain;
+
+ SavingTrustManager(X509TrustManager tm) {
+ this.tm = tm;
+ }
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void checkClientTrusted(X509Certificate[] chain, String authType)
+ throws CertificateException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] chain, String authType)
+ throws CertificateException {
+ this.chain = chain;
+ tm.checkServerTrusted(chain, authType);
+ }
+ }
+
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index bed17ec..7fbc674 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -8,7 +8,6 @@ spring:
url: jdbc:mysql://127.0.0.1:3306/?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=Asia/Shanghai&autoReconnect=true
driver-class-name: com.mysql.cj.jdbc.Driver
hikari:
- max-lifetime: 600000
connection-test-query: SELECT 1
connection-timeout: 10000
minimum-idle: 2
diff --git a/src/main/resources/mapping/TemplateMapper.xml b/src/main/resources/mapping/TemplateMapper.xml
index 542b285..a441535 100644
--- a/src/main/resources/mapping/TemplateMapper.xml
+++ b/src/main/resources/mapping/TemplateMapper.xml
@@ -42,4 +42,8 @@
#{id}
+
diff --git a/src/main/resources/static/ddt.html b/src/main/resources/static/ddt.html
index e420dbd..369c145 100644
--- a/src/main/resources/static/ddt.html
+++ b/src/main/resources/static/ddt.html
@@ -83,6 +83,9 @@
+
+
+
@@ -153,7 +156,7 @@
有效期 |
templateId |
pic |
- 竞拍 |
+ 竞拍 |
diff --git a/src/main/resources/static/js.js b/src/main/resources/static/js.js
index 07f80ad..1b7961d 100644
--- a/src/main/resources/static/js.js
+++ b/src/main/resources/static/js.js
@@ -130,6 +130,7 @@ function getAuctionItems() {
let itemName = document.getElementById("itemName").value;
let selfId = document.getElementById("selfId").value;
let userId = document.getElementById("userId").value;
+ let buyerId = document.getElementById("buyerId").value;
let orderTime = document.getElementById("orderTime");
let orderName = document.getElementById("orderName");
@@ -168,7 +169,8 @@ function getAuctionItems() {
sort: sort,
itemName: itemName,
userId: userId,
- selfId: selfId
+ selfId: selfId,
+ buyerId: buyerId
}
}).then(res => {
// 接口数据
@@ -295,79 +297,69 @@ function Item(auctionID, auctioneerID, auctioneerName, beginTimeString, buyerID,
this.userDefinePrice = userDefinePrice;
}
+function buildTableHtml(item, i) {
+ let bidUnitColor = "bid-unit-black";
+ let mouthUnitColor = "mouth-unit-black";
+ let buyStatus = "has-not-buyer";
+ let userDefinePrice = "";
-function buildTable(items) {
- let tableHtml = '' +
- ' \n' +
- ' | 序号 | \n' +
- ' 物品名称 | \n' +
- ' 数量 | \n' +
- ' 竞拍价 | \n' +
- ' 一口价 | \n' +
- ' 竞拍价单价 | \n' +
- ' 一口价单价 | \n' +
- ' 期望价格 | \n' +
- ' 出售者 | \n' +
- ' 时间 | \n' +
- ' 有效期 | \n' +
- ' templateId | \n' +
- ' pic | \n' +
- ' 竞拍 | \n' +
- '
';
-
- let resultBody = document.getElementById("result-body");
- resultBody.innerHTML = "";
- if (items == null || items.length === 0) {
- return;
- }
-
- let innerHtml = "";
- let tid = items[0].templateId;
- let tidSame = true;
- for (let i = 0; i < items.length; i++) {
- let item = items[i];
+ if (item == null) {
+ item = new Item();
+ } else {
item.unitPrice = parseFloat(item.unitPrice.toFixed(2));
item.unitMouthfulPrice = parseFloat(item.unitMouthfulPrice.toFixed(2));
- let htmlTemp = tableHtml
- .replace("序号", i)
- .replace("物品名称", item.templateName)
- .replace("数量", item.count)
- .replace("竞拍价", item.price)
- .replace("一口价", item.mouthful)
- .replace("竞拍价单价", item.unitPrice)
- .replace("一口价单价", item.unitMouthfulPrice)
- .replace("期望价格", item.userDefinePrice == null ? "" : item.userDefinePrice)
- .replace("出售者", item.auctioneerName)
- .replace("时间", item.beginTimeString)
- .replace("有效期", item.validDate)
- .replace("templateId", item.templateId)
- .replace("pic", item.pic)
- .replace("竞拍", item.buyerName)
- ;
+
if (item.userDefinePrice != null) {
+ userDefinePrice = item.userDefinePrice;
if (item.unitPrice > 0 && item.userDefinePrice > item.unitPrice) {
- console.log(item.userDefinePrice);
- console.log(item.unitPrice);
- console.log(item.userDefinePrice > item.unitPrice);
- console.log()
- htmlTemp = htmlTemp.replace("bid-unit-black", "bid-unit-red");
+ bidUnitColor = "bid-unit-red";
}
if (item.unitMouthfulPrice > 0 && item.userDefinePrice > item.unitMouthfulPrice) {
- htmlTemp = htmlTemp.replace("mouth-unit-black", "mouth-unit-red");
+ mouthUnitColor = "mouth-unit-red";
}
-
-
}
if (item.buyerName != null && item.buyerName !== "") {
- htmlTemp = htmlTemp.replace("has-not-buyer", "has-buyer");
+ buyStatus = "has-buyer";
}
- if (item.templateId !== tid) {
+ }
+
+ return `
+ \n
+ | ${i} |
+ ${item.templateName} |
+ ${item.count} |
+ ${item.price} |
+ ${item.mouthful} |
+ ${item.unitPrice} |
+ ${item.unitMouthfulPrice} |
+ ${userDefinePrice} |
+ ${item.auctioneerName} |
+ ${item.beginTimeString} |
+ ${item.validDate} |
+ ${item.templateId} |
+ ${item.pic} |
+ ${item.buyerName} |
+
`;
+}
+
+function buildTable(items) {
+ let resultBody = document.getElementById("result-body");
+ resultBody.innerHTML = "";
+ if (items == null || items.length === 0) {
+ return;
+ }
+ let innerHtml = "";
+ let tid = items[0].templateId;
+ let tidSame = true;
+ for (let i = 0; i < items.length; i++) {
+ let item = items[i];
+ if (item.templateId !== tid) {
tidSame = false;
}
- innerHtml += htmlTemp;
+ innerHtml += buildTableHtml(item, i);
}
if (tidSame) {
document.getElementById("template-id").value = tid;
@@ -377,4 +369,3 @@ function buildTable(items) {
resultBody.innerHTML = innerHtml;
}
-