[dev] Add customize fingerprint

src/scrapy_redis/dupefilter.py

@@ -1,8 +1,12 @@
 import logging
+import hashlib
+import json
 import time
 
 from scrapy.dupefilters import BaseDupeFilter
 from scrapy.utils.request import request_fingerprint
+from scrapy.utils.python import to_unicode
+from w3lib.url import canonicalize_url
 
 from . import defaults
 from .connection import get_redis_from_settings
@@ -112,8 +116,14 @@ def request_fingerprint(self, request):
         str
 
         """
-        return request_fingerprint(request)
-
+        fingerprint_data = {
+            "method": to_unicode(request.method),
+            "url": canonicalize_url(request.url),
+            "body": (request.body or b"").hex(),
+        }
+        fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+        return hashlib.sha1(fingerprint_json.encode()).hexdigest()
+
     @classmethod
     def from_spider(cls, spider):
         settings = spider.settings

tests/test_dupefilter.py

@@ -11,8 +11,8 @@ def get_redis_mock():
 
     def sadd(key, fp, added=0, db={}):
         fingerprints = db.setdefault(key, set())
-        if key not in fingerprints:
-            fingerprints.add(key)
+        if fp not in fingerprints:
+            fingerprints.add(fp)
             added += 1
         return added
 
@@ -30,8 +30,24 @@ def setup(self):
 
     def test_request_seen(self):
         req = Request('http://example.com')
-        assert not self.df.request_seen(req)
-        assert self.df.request_seen(req)
+
+        def same_request():
+            assert not self.df.request_seen(req)
+            assert self.df.request_seen(req)
+
+        def diff_method():
+            diff_method = Request('http://example.com', method='POST')
+            assert self.df.request_seen(req)
+            assert not self.df.request_seen(diff_method)
+
+        def diff_url():
+            diff_url = Request('http://example2.com')
+            assert self.df.request_seen(req)
+            assert not self.df.request_seen(diff_url)
+
+        same_request()
+        diff_method()
+        diff_url()
 
     def test_overridable_request_fingerprinter(self):
         req = Request('http://example.com')