Ausgabe der neuen DB Einträge

venv/lib/python3.9/site-packages/scrapy/utils/signal.py

@@ -0,0 +1,80 @@
+"""Helper functions for working with signals"""
+
+import logging
+
+from twisted.internet.defer import DeferredList, Deferred
+from twisted.python.failure import Failure
+
+from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
+from pydispatch.robustapply import robustApply
+
+from scrapy.exceptions import StopDownload
+from scrapy.utils.defer import maybeDeferred_coro
+from scrapy.utils.log import failure_to_exc_info
+
+
+logger = logging.getLogger(__name__)
+
+
+class _IgnoredException(Exception):
+    pass
+
+
+def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
+    """Like pydispatcher.robust.sendRobust but it also logs errors and returns
+    Failures instead of exceptions.
+    """
+    dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
+    spider = named.get('spider', None)
+    responses = []
+    for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        try:
+            response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
+            if isinstance(response, Deferred):
+                logger.error("Cannot return deferreds from signal handler: %(receiver)s",
+                             {'receiver': receiver}, extra={'spider': spider})
+        except dont_log:
+            result = Failure()
+        except Exception:
+            result = Failure()
+            logger.error("Error caught on signal handler: %(receiver)s",
+                         {'receiver': receiver},
+                         exc_info=True, extra={'spider': spider})
+        else:
+            result = response
+        responses.append((receiver, result))
+    return responses
+
+
+def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
+    """Like send_catch_log but supports returning deferreds on signal handlers.
+    Returns a deferred that gets fired once all signal handlers deferreds were
+    fired.
+    """
+    def logerror(failure, recv):
+        if dont_log is None or not isinstance(failure.value, dont_log):
+            logger.error("Error caught on signal handler: %(receiver)s",
+                         {'receiver': recv},
+                         exc_info=failure_to_exc_info(failure),
+                         extra={'spider': spider})
+        return failure
+
+    dont_log = named.pop('dont_log', None)
+    spider = named.get('spider', None)
+    dfds = []
+    for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
+        d.addErrback(logerror, receiver)
+        d.addBoth(lambda result: (receiver, result))
+        dfds.append(d)
+    d = DeferredList(dfds)
+    d.addCallback(lambda out: [x[1] for x in out])
+    return d
+
+
+def disconnect_all(signal=Any, sender=Any):
+    """Disconnect all signal handlers. Useful for cleaning up after running
+    tests
+    """
+    for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        disconnect(receiver, signal=signal, sender=sender)