Ausgabe der neuen DB Einträge

venv/lib/python3.9/site-packages/scrapy/downloadermiddlewares/downloadtimeout.py

@@ -0,0 +1,26 @@
+"""
+Download timeout middleware
+
+See documentation in docs/topics/downloader-middleware.rst
+"""
+
+from scrapy import signals
+
+
+class DownloadTimeoutMiddleware:
+
+    def __init__(self, timeout=180):
+        self._timeout = timeout
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
+        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+        return o
+
+    def spider_opened(self, spider):
+        self._timeout = getattr(spider, 'download_timeout', self._timeout)
+
+    def process_request(self, request, spider):
+        if self._timeout:
+            request.meta.setdefault('download_timeout', self._timeout)