Source code for scrapy.downloadermiddlewares.downloadtimeout

"""
Download timeout middleware

See documentation in docs/topics/downloader-middleware.rst
"""

from scrapy import signals


[docs]class DownloadTimeoutMiddleware: def __init__(self, timeout=180): self._timeout = timeout @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT")) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o def spider_opened(self, spider): self._timeout = getattr(spider, "download_timeout", self._timeout) def process_request(self, request, spider): if self._timeout: request.meta.setdefault("download_timeout", self._timeout)