scrapy中deferred的回调

def _next_request_from_scheduler(self, spider):#在引擎中处理一个请求
        slot = self.slot
        request = slot.scheduler.next_request()
        if not request:
            return
        d = self._download(request, spider)#生成了一个deferred对象
        d.addBoth(self._handle_downloader_output, request, spider)
        d.addErrback(lambda f: logger.info('Error while handling downloader output',
                                           exc_info=failure_to_exc_info(f),
                                           extra={'spider': spider}))
        d.addBoth(lambda _: slot.remove_request(request))
        d.addErrback(lambda f: logger.info('Error while removing request from slot',
                                           exc_info=failure_to_exc_info(f),
                                           extra={'spider': spider}))
        d.addBoth(lambda _: slot.nextcall.schedule())
        d.addErrback(lambda f: logger.info('Error while scheduling new request',
                                           exc_info=failure_to_exc_info(f),
                                           extra={'spider': spider}))
        return d
 
def _download(self, request, spider):
        slot = self.slot
        slot.add_request(request)
        def _on_success(response):
            assert isinstance(response, (Response, Request))
            if isinstance(response, Response):
                response.request = request # tie request to response received
                logkws = self.logformatter.crawled(request, response, spider)
                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
                self.signals.send_catch_log(signal=signals.response_received, \
                    response=response, request=request, spider=spider)
            return response
 
      def _on_complete(_):
            slot.nextcall.schedule()
            return _
 
        dwld = self.downloader.fetch(request, spider)#下载器fetch
        dwld.addCallbacks(_on_success)
        dwld.addBoth(_on_complete)
        return dwld
在HTTP11处理器中
class HTTP11DownloadHandler(object):
 
def download_request(self, request, spider):
        """Return a deferred for the HTTP download"""
        agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
            fail_on_dataloss=self._fail_on_dataloss)
        return agent.download_request(request)
 
class ScrapyAgent(object):
 
def download_request(self, request):
        timeout = request.meta.get('download_timeout') or self._connectTimeout
        agent = self._get_agent(request, timeout)
 
        # request details
        url = urldefrag(request.url)[0]
        method = to_bytes(request.method)
        headers = TxHeaders(request.headers)
        if isinstance(agent, self._TunnelingAgent):
            headers.removeHeader(b'Proxy-Authorization')
        if request.body:
            bodyproducer = _RequestBodyProducer(request.body)
        elif method == b'POST':
            # Setting Content-Length: 0 even for POST requests is not a
            # MUST per HTTP RFCs, but it's common behavior, and some
            # servers require this, otherwise returning HTTP 411 Length required
            #
            # RFC 7230#section-3.3.2:
            # "a Content-Length header field is normally sent in a POST
            # request even when the value is 0 (indicating an empty payload body)."
            #
            # Twisted < 17 will not add "Content-Length: 0" by itself;
            # Twisted >= 17 fixes this;
            # Using a producer with an empty-string sends `0` as Content-Length
            # for all versions of Twisted.
            bodyproducer = _RequestBodyProducer(b'')
        else:
            bodyproducer = None
        start_time = time()
        d = agent.request(
            method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
        # set download latency
        d.addCallback(self._cb_latency, request, start_time)
        # response body is ready to be consumed
        d.addCallback(self._cb_bodyready, request)
        d.addCallback(self._cb_bodydone, request, url)
        # check download timeout
        self._timeout_cl = reactor.callLater(timeout, d.cancel)
        d.addBoth(self._cb_timeout, request, url, timeout)
        return d
 
class _ResponseReader(protocol.Protocol):
 
    def __init__(self, finished, txresponse, request, maxsize, warnsize,
                 fail_on_dataloss):
        self._finished = finished
        self._txresponse = txresponse
        self._request = request
        self._bodybuf = BytesIO()
        self._maxsize  = maxsize
        self._warnsize  = warnsize
        self._fail_on_dataloss = fail_on_dataloss
        self._fail_on_dataloss_warned = False
        self._reached_warnsize = False
        self._bytes_received = 0
 
    def dataReceived(self, bodyBytes):#读取数据，放到缓冲
        # This maybe called several times after cancel was called with buffered
        # data.
        if self._finished.called:
            return
 
        self._bodybuf.write(bodyBytes)
        self._bytes_received += len(bodyBytes)
 
        if self._maxsize and self._bytes_received > self._maxsize:
            logger.error("Received (%(bytes)s) bytes larger than download "
                         "max size (%(maxsize)s).",
                         {'bytes': self._bytes_received,
                          'maxsize': self._maxsize})
            # Clear buffer earlier to avoid keeping data in memory for a long
            # time.
            self._bodybuf.truncate(0)
            self._finished.cancel()
 
        if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
            self._reached_warnsize = True
            logger.warning("Received more bytes than download "
                           "warn size (%(warnsize)s) in request %(request)s.",
                           {'warnsize': self._warnsize,
                            'request': self._request})
 
    def connectionLost(self, reason):#连接完成后调用，也即响应已经到达。
        if self._finished.called:
            return
 
        body = self._bodybuf.getvalue()
        if reason.check(ResponseDone):#self._finished是deferred对象
            self._finished.callback((self._txresponse, body, None))#回调了，
            return
 
        if reason.check(PotentialDataLoss):
            self._finished.callback((self._txresponse, body, ['partial']))
            return
 
        if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
            if not self._fail_on_dataloss:
                self._finished.callback((self._txresponse, body, ['dataloss']))
                return
 
            elif not self._fail_on_dataloss_warned:
                logger.warn("Got data loss in %s. If you want to process broken "
                            "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
                            " -- This message won't be shown in further requests",
                            self._txresponse.request.absoluteURI.decode())
                self._fail_on_dataloss_warned = True
 
        self._finished.errback(reason)