scrapy-redis 之处理异常

今天心情不好 不想多打字 自己看注释吧

 from scrapy.http import HtmlResponse
 from twisted.internet import defer
 from twisted.internet.error import TimeoutError, DNSLookupError, \
     ConnectionRefusedError, ConnectionDone, ConnectError, \
     ConnectionLost, TCPTimedOutError
 from twisted.web.client import ResponseFailed
 from scrapy.core.downloader.handlers.http11 import TunnelError

 class ProcessAllExceptionMiddleware(object):
     ALL_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
                       ConnectionRefusedError, ConnectionDone, ConnectError,
                       ConnectionLost, TCPTimedOutError, ResponseFailed,
                       IOError, TunnelError)

     def process_response(self, request, response, spider):
         # 捕获状态码为40x/50x的response
         if str(response.status).startswith('') or str(response.status).startswith(''):
             # 随意封装，直接返回response，spider代码中根据url==''来处理response
             response = HtmlResponse(url=str(response.status), status=200)
             return response
         # 其他状态码不处理
         return response

     def process_exception(self, request, exception, spider):
         # 捕获几乎所有的异常
         if isinstance(exception, self.ALL_EXCEPTIONS):
             # 在日志中打印异常类型
             print('Got exception: %s' % (exception))
             # 随意封装一个response，返回给spider
             response = HtmlResponse(url='exception')
             return response
         # 打印出未捕获到的异常
         print('not contained exception: %s' % exception)

然后根据返回的url不同就可以在spider中进行各种处理了