Skip to content

Commit

Permalink
Fixed issue when passing errback parameter
Browse files Browse the repository at this point in the history
  • Loading branch information
pedrosalgueiro committed Jul 22, 2019
1 parent d8bce51 commit 75d2b3e
Showing 1 changed file with 11 additions and 2 deletions.
13 changes: 11 additions & 2 deletions scrapyrt/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,10 @@ def __init__(self, spider_name, request_kwargs, max_requests=None, start_request
# callback will be added after instantiation of crawler object
# because we need to know if spider has method available
self.callback_name = request_kwargs.pop('callback', None) or 'parse'

# do the same for errback
self.errback_name = request_kwargs.pop('errback', None) or 'parse'

if request_kwargs.get("url"):
self.request = self.create_spider_request(deepcopy(request_kwargs))
else:
Expand Down Expand Up @@ -185,8 +189,8 @@ def spider_idle(self, spider):
"""Handler of spider_idle signal.
Schedule request for url given to api, with optional callback
that can be passed as GET parameter.
and errback that can be passed as GET parameter.
spider_idle signal is used because we want to optionally enable
start_requests for the spider and if request is scheduled in
spider_opened signal handler it's fired earlier then start_requests
Expand All @@ -197,6 +201,11 @@ def spider_idle(self, spider):
callback = getattr(self.crawler.spider, self.callback_name)
assert callable(callback), 'Invalid callback'
self.request = self.request.replace(callback=callback)

errback = getattr(self.crawler.spider, self.errback_name)
assert callable(errback), 'Invalid errback'
self.request = self.request.replace(errback=errback)

modify_request = getattr(
self.crawler.spider, "modify_realtime_request", None)
if callable(modify_request):
Expand Down

0 comments on commit 75d2b3e

Please sign in to comment.