@dev 优化查询

sleepybear1113 · Aug 23, 2021 · 6653872 · 6653872
1 parent f53dadb
commit 6653872
Show file tree

Hide file tree

Showing 26 changed files with 506 additions and 95 deletions.
diff --git a/pom.xml b/pom.xml
@@ -10,7 +10,7 @@
     </parent>
     <groupId>com.xjx</groupId>
     <artifactId>ddt-crawler</artifactId>
-    <version>0.0.3-SNAPSHOT</version>
+    <version>0.0.5-SNAPSHOT</version>
     <name>ddt-crawler</name>
     <description>ddt-crawler</description>
     <properties>

diff --git a/src/main/java/com/xjx/ddtcrawler/cache/CacheDomain.java b/src/main/java/com/xjx/ddtcrawler/cache/CacheDomain.java
@@ -4,14 +4,25 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import java.io.Serializable;
+
 /**
  * @author XieJiaxing
  * @date 2021/8/7 17:36
  */
 @Data
 @NoArgsConstructor
 @AllArgsConstructor
-public class CacheDomain {
+public class CacheDomain implements Serializable {
+    private static final long serialVersionUID = -1617868566077296126L;
+
     private Object object;
     private Long expireAt;
+
+    public boolean isExpired() {
+        if (expireAt == null) {
+            return false;
+        }
+        return expireAt < System.currentTimeMillis();
+    }
 }
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/CacheInterface.java b/src/main/java/com/xjx/ddtcrawler/cache/CacheInterface.java
@@ -0,0 +1,12 @@
+package com.xjx.ddtcrawler.cache;
+
+/**
+ * @author XieJiaxing
+ * @date 2021/8/21 17:37
+ */
+public interface CacheInterface {
+    /**
+     * 清理过期的
+     */
+    void clearExpired();
+}
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/CommonCache.java b/src/main/java/com/xjx/ddtcrawler/cache/CommonCache.java
@@ -1,5 +1,6 @@
 package com.xjx.ddtcrawler.cache;
 
+import org.apache.commons.collections4.MapUtils;
 import org.springframework.stereotype.Component;
 
 import java.util.Map;
@@ -10,10 +11,13 @@
  * @date 2021/8/7 17:33
  */
 @Component
-public class CommonCache {
+public class CommonCache implements CacheInterface {
     private final Map<String, CacheDomain> CACHE_MAP = new ConcurrentHashMap<>();
 
     public void setCache(String key, Object value, Long expireTime) {
+        if (value == null) {
+            return;
+        }
         Long expireAt = null;
         if (expireTime != null) {
             expireAt = System.currentTimeMillis() + expireTime;
@@ -49,4 +53,13 @@ public <T> T getCache(String key) {
             return null;
         }
     }
+
+    @Override
+    public void clearExpired() {
+        if (MapUtils.isEmpty(CACHE_MAP)) {
+            return;
+        }
+
+        CACHE_MAP.values().removeIf(CacheDomain::isExpired);
+    }
 }
diff --git a/src/main/java/com/xjx/ddtcrawler/cache/WebUserCache.java b/src/main/java/com/xjx/ddtcrawler/cache/WebUserCache.java
@@ -1,7 +1,7 @@
 package com.xjx.ddtcrawler.cache;
 
 import com.xjx.ddtcrawler.cookie.WebUser;
-import com.xjx.ddtcrawler.utils.EncryptedUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
@@ -13,7 +13,7 @@
  * @date 2021/8/1 22:56
  */
 @Component
-public class WebUserCache {
+public class WebUserCache implements CacheInterface {
     private final static Map<String, WebUser> MAP = new ConcurrentHashMap<>();
 
     public WebUser getById(String id) {
@@ -25,7 +25,7 @@ public WebUser getById(String id) {
             return null;
         }
 
-        Long expireTimeAt = webUser.getExpireTimeAt();
+        Long expireTimeAt = webUser.getExpireAt();
         if (expireTimeAt == null) {
             return webUser;
         } else {
@@ -49,7 +49,7 @@ public String saveUser(WebUser webUser) {
 
         Long userId = webUser.getUserId();
         String key = webUser.getKey();
-        Long expireTimeAt = webUser.getExpireTimeAt();
+        Long expireTimeAt = webUser.getExpireAt();
         if (userId == null || StringUtils.isBlank(key)) {
             return null;
         }
@@ -84,4 +84,13 @@ public void delete(String id) {
 
         MAP.remove(id);
     }
+
+    @Override
+    public void clearExpired() {
+        if (MapUtils.isEmpty(MAP)) {
+            return;
+        }
+
+        MAP.values().removeIf(CacheDomain::isExpired);
+    }
 }
diff --git a/src/main/java/com/xjx/ddtcrawler/controller/AuctionController.java b/src/main/java/com/xjx/ddtcrawler/controller/AuctionController.java
@@ -24,6 +24,7 @@ public class AuctionController {
     @RequestMapping("/auction/getAuctionItems")
     public Result getItems(@RequestParam(required = false, defaultValue = "1") Integer page,
                            @RequestParam(required = false, defaultValue = "-1") Long userId,
+                           @RequestParam(required = false, defaultValue = "-1") Long buyerId,
                            @RequestParam(required = false, defaultValue = "") String itemName,
                            @RequestParam(required = false, defaultValue = "2") Integer order,
                            @RequestParam(required = false, defaultValue = "true") Boolean sort) throws MyException, InterruptedException {
@@ -41,11 +42,13 @@ public Result getItems(@RequestParam(required = false, defaultValue = "1") Integ
         queryUrl.setWebUser(webUser);
         queryUrl.setOrder(order);
         queryUrl.setUserId(userId);
+        queryUrl.setBuyId(buyerId);
         queryUrl.setSort(sort);
         queryUrl.setPage(page);
         queryUrl.setName(itemName.trim());
         if (temporaryUser) {
             queryUrl.setUserId(-1L);
+            queryUrl.setBuyId(-1L);
         }
 
         Result result = auctionLogic.getSingleResult(queryUrl);
@@ -71,7 +74,7 @@ public Result getAuctionPriceOder(String itemName,
         queryUrl.setSort(sort);
         queryUrl.setName(itemName.trim());
 
-        Result result = auctionLogic.getResultsByBatchPages(queryUrl, QueryUrl.DEFAULT_PAGES, true, 250L, priceType, sort);
+        Result result = auctionLogic.getResultsByBatchPages(queryUrl, QueryUrl.DEFAULT_PAGES, true, 250L, priceType, sort, null);
         if (webUser.isTemporaryUser()) {
             result.hideSensitiveInfo();
         }

diff --git a/src/main/java/com/xjx/ddtcrawler/controller/TemplateController.java b/src/main/java/com/xjx/ddtcrawler/controller/TemplateController.java
@@ -8,6 +8,8 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.List;
+
 /**
  * @author XieJiaxing
  * @date 2021/8/14 18:33
@@ -26,7 +28,6 @@ public Template getTemplateById(Long id) throws MyException {
         return templateLogic.getTemplateById(id);
     }
 
-
     @RequestMapping("/template/saveTemplate")
     public Boolean saveTemplate(Long templateId, String templateName) throws MyException {
         WebUser webUser = WebUser.getSafeWebUser();
@@ -35,4 +36,14 @@ public Boolean saveTemplate(Long templateId, String templateName) throws MyExcep
         }
         return templateLogic.saveTemplate(templateId, templateName);
     }
+
+    @RequestMapping("/template/listAllTemplates")
+    public List<Template> listAllTemplates() {
+        return templateLogic.listAllTemplates();
+    }
+
+    @RequestMapping("/template/listCommonSlv4")
+    public List<Template> listCommonSlv4() {
+        return templateLogic.listCommonSlv4();
+    }
 }
diff --git a/src/main/java/com/xjx/ddtcrawler/controller/UserController.java b/src/main/java/com/xjx/ddtcrawler/controller/UserController.java
@@ -51,13 +51,16 @@ public MyMessage createTemporaryLicense(Long expireTime, @RequestParam(required
         }
 
         long minConcurrentTime = 100L;
+        if (concurrentTime < webUser.getConcurrentTime()) {
+            concurrentTime = webUser.getConcurrentTime();
+        }
         if (concurrentTime < minConcurrentTime) {
             concurrentTime = minConcurrentTime;
         }
 
         long now = System.currentTimeMillis();
         if (expireTime != null) {
-            webUser.setExpireTimeAt(expireTime * 1000 + now);
+            webUser.setExpireAt(expireTime * 1000 + now);
         }
         webUser.setConcurrentTime(concurrentTime);
         String temporaryLicense = webUserLogic.generateTemporaryLicense(webUser);

diff --git a/src/main/java/com/xjx/ddtcrawler/cookie/WebUser.java b/src/main/java/com/xjx/ddtcrawler/cookie/WebUser.java
@@ -1,8 +1,10 @@
 package com.xjx.ddtcrawler.cookie;
 
+import com.xjx.ddtcrawler.cache.CacheDomain;
 import com.xjx.ddtcrawler.exception.MyException;
 import com.xjx.ddtcrawler.utils.EncryptedUtils;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.BeanUtils;
 
@@ -12,8 +14,9 @@
  * @author XieJiaxing
  * @date 2021/8/14 12:29
  */
+@EqualsAndHashCode(callSuper = true)
 @Data
-public class WebUser implements Serializable {
+public class WebUser extends CacheDomain implements Serializable {
     private static final long serialVersionUID = 5392034256168678992L;
 
     public static final ThreadLocal<WebUser> WEB_USER_THREAD_LOCAL = new ThreadLocal<>();
@@ -22,12 +25,11 @@ public class WebUser implements Serializable {
     private Long loginTime;
     private Long userId;
     private String key;
-    private Long expireTimeAt;
     private String temporaryLicense;
     /**
      * 用户并发毫秒
      */
-    private Long concurrentTime = 1000L;
+    private Long concurrentTime = 500L;
 
     public static void setWebUser(WebUser webUser) {
         WEB_USER_THREAD_LOCAL.set(webUser);
@@ -42,7 +44,7 @@ public static WebUser getSafeWebUser() throws MyException {
         if (webUser == null) {
             throw new MyException("用户不存在或者已过期");
         }
-        Long expireTimeAt = webUser.getExpireTimeAt();
+        Long expireTimeAt = webUser.getExpireAt();
         if (expireTimeAt != null && expireTimeAt < System.currentTimeMillis()) {
             throw new MyException("用户不存在或者已过期");
         }

diff --git a/src/main/java/com/xjx/ddtcrawler/domain/Item.java b/src/main/java/com/xjx/ddtcrawler/domain/Item.java
@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.thoughtworks.xstream.annotations.XStreamAlias;
 import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+import com.xjx.ddtcrawler.utils.EncryptedUtils;
 import com.xjx.ddtcrawler.utils.TimeUtil;
 import lombok.Data;
 
@@ -162,6 +163,29 @@ public class Item implements Serializable {
      * 用户定义的低价
      */
     private Double userDefinePrice;
+    private Boolean isTemplateIdEncrypted = false;
+
+    public void encryptTemplateId() {
+        if (this.isTemplateIdEncrypted == null) {
+            this.isTemplateIdEncrypted = false;
+        }
+        if (this.isTemplateIdEncrypted) {
+            return;
+        }
+        this.templateId = EncryptedUtils.encryptTemplateId(this.templateId);
+        this.isTemplateIdEncrypted = true;
+    }
+
+    public void decryptTemplateId() {
+        if (this.isTemplateIdEncrypted == null) {
+            this.isTemplateIdEncrypted = false;
+        }
+        if (!this.isTemplateIdEncrypted) {
+            return;
+        }
+        this.templateId = EncryptedUtils.decryptTemplateId(this.templateId);
+        this.isTemplateIdEncrypted = false;
+    }
 
     @Override
     public String toString() {

diff --git a/src/main/java/com/xjx/ddtcrawler/domain/QueryUrl.java b/src/main/java/com/xjx/ddtcrawler/domain/QueryUrl.java
@@ -98,7 +98,9 @@ public String buildUrl() throws MyException {
         rnd = RANDOM.nextDouble();
         String format = "?page=%s&selfid=%s&type=%s&Auctions=%s&buyID=%s&key=%s&name=%s&userId=%s&rnd=%s&pay=%s&order=%s&sort=%s";
 
-        return URL + String.format(format, page, selfId, type, auctions, buyId, key, ec(name), userId, rnd, pay, order, sort);
+        String url = URL + String.format(format, page, selfId, type, auctions, buyId, key, ec(name), userId, rnd, pay, order, sort);
+        log.info(url);
+        return url;
     }
 
     /**

diff --git a/src/main/java/com/xjx/ddtcrawler/domain/Result.java b/src/main/java/com/xjx/ddtcrawler/domain/Result.java
@@ -95,7 +95,6 @@ public void hideSensitiveInfo() {
                 item.setBuyerName("有");
             }
             item.setAuctioneerId(null);
-            item.setTemplateId(null);
             item.setBeginDate(null);
             item.setAuctionDate(null);
             item.setItemDate(null);
@@ -104,6 +103,7 @@ public void hideSensitiveInfo() {
             item.setUserDefinePrice(null);
             Long beginTime = item.getBeginTime();
             item.setBeginTime(null);
+            item.encryptTemplateId();
 
             if (beginTime != null) {
                 long v = (System.currentTimeMillis() - beginTime) / 1000 / 3600;

diff --git a/src/main/java/com/xjx/ddtcrawler/domain/Template.java b/src/main/java/com/xjx/ddtcrawler/domain/Template.java
@@ -1,5 +1,6 @@
 package com.xjx.ddtcrawler.domain;
 
+import com.xjx.ddtcrawler.utils.EncryptedUtils;
 import lombok.Data;
 
 import java.io.Serializable;
@@ -16,4 +17,21 @@ public class Template implements Serializable {
     private String name;
     private Double price;
     private Long modifyTime;
+    private Boolean isEncrypted = false;
+
+    public void encryptId() {
+        if (this.isEncrypted) {
+            return;
+        }
+        this.id = EncryptedUtils.encryptTemplateId(this.id);
+        this.isEncrypted = true;
+    }
+
+    public void decryptId() {
+        if (!this.isEncrypted) {
+            return;
+        }
+        this.id = EncryptedUtils.decryptTemplateId(this.id);
+        this.isEncrypted = false;
+    }
 }