Dev debug pytest (#230)

* add import hint for pytest

* update pytest usage

* update deprecated scrapy.utils.request usage

* Update .gitignore & LICENSE  (#225)

* update .gitignore & LICENSE

* remove contribution section and add alternative choice

* [docs] Remove docs $ prefix (#229)

* remove docs $ prefix

* align code indent

* add import hint for pytest

* add import hint for pytest

* add text color helper

* add json type check

add json formatted_data type check and warning message

* fix subset not equal assert

* update test & install guide

CONTRIBUTING.rst

@@ -79,18 +79,28 @@ Ready to contribute? Here's how to set up `scrapy-redis` for local development.
 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox::
 
        flake8 scrapy_redis tests
-       pytest --ignore=setup.py
+       pip install .
+       python -m pytest --ignore=setup.py
        tox
 
    To get flake8 and tox, just pip install them into your virtualenv.
 
-6. Commit your changes and push your branch to GitHub::
+6. Note that if the error of `No module named scrapy_redis` shows, please install `scrapy-redis` of your branch by::
+   
+       pip install .
+
+7. Or change the import lines::
+
+       from scrapy_redis import xxx # from this
+       from src.scrapy_redis import xxx # to this
+
+8. Commit your changes and push your branch to GitHub::
 
        git add .
        git commit -m "Your detailed description of your changes."
        git push origin name-of-your-bugfix-or-feature
 
-7. Submit a pull request through the GitHub website.
+9. Submit a pull request through the GitHub website.
 
 Pull Request Guidelines
 -----------------------

src/scrapy_redis/queue.py

@@ -1,4 +1,4 @@
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy.utils.request import request_from_dict
 
 from . import picklecompat
 
@@ -39,13 +39,13 @@ def __init__(self, server, spider, key, serializer=None):
 
     def _encode_request(self, request):
         """Encode a request object"""
-        obj = request_to_dict(request, self.spider)
+        obj = request.to_dict(spider=self.spider)
         return self.serializer.dumps(obj)
 
     def _decode_request(self, encoded_request):
         """Decode an request previously encoded"""
         obj = self.serializer.loads(encoded_request)
-        return request_from_dict(obj, self.spider)
+        return request_from_dict(obj, spider=self.spider)
 
     def __len__(self):
         """Return the length of the queue"""

src/scrapy_redis/spiders.py

@@ -3,6 +3,7 @@
 from scrapy import signals, FormRequest
 from scrapy.exceptions import DontCloseSpider
 from scrapy.spiders import Spider, CrawlSpider
+from scrapy_redis import TextColor
 import time
 
 from . import connection, defaults
@@ -169,7 +170,14 @@ def make_request_from_data(self, data):
         formatted_data = bytes_to_str(data, self.redis_encoding)
 
         # change to json array
-        parameter = json.loads(formatted_data)
+        parameter = {}
+        if type(formatted_data) == dict:
+            parameter = json.loads(formatted_data)
+        else:
+            print(TextColor.WARNING + "WARNING: String request is deprecated, please use JSON data format. \
+                Detail information, please check https://github.com/rmax/scrapy-redis#features" + TextColor.ENDC)
+            return FormRequest(formatted_data, dont_filter=True)
+
         url = parameter['url']
         del parameter['url']
         metadata = {}

src/scrapy_redis/utils.py

@@ -1,6 +1,17 @@
 import six
 
 
+class TextColor:
+    HEADER = '\033[95m'
+    OKBLUE = '\033[94m'
+    OKCYAN = '\033[96m'
+    OKGREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
+    BOLD = '\033[1m'
+    UNDERLINE = '\033[4m'
+
 def bytes_to_str(s, encoding='utf-8'):
     """Returns a str if a bytes object is given."""
     if six.PY3 and isinstance(s, bytes):

tests/test_spiders.py

@@ -148,7 +148,7 @@ def test_consume_urls_from_redis(start_urls_as_zset, start_urls_as_set, spider_c
         start_requests = list(spider.start_requests())
         if start_urls_as_zset or start_urls_as_set:
             assert len(start_requests) == batch_size
-            assert set(start_requests).issubset(reqs)
+            assert set(map(lambda x: x.url, start_requests)).issubset(map(lambda x: x.url, reqs))
         else:
             assert start_requests == reqs[:batch_size]