80 lines
3 KiB
Python
80 lines
3 KiB
Python
"""Helper functions for working with signals"""
|
|
|
|
import logging
|
|
|
|
from twisted.internet.defer import DeferredList, Deferred
|
|
from twisted.python.failure import Failure
|
|
|
|
from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers
|
|
from pydispatch.robustapply import robustApply
|
|
|
|
from scrapy.exceptions import StopDownload
|
|
from scrapy.utils.defer import maybeDeferred_coro
|
|
from scrapy.utils.log import failure_to_exc_info
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class _IgnoredException(Exception):
|
|
pass
|
|
|
|
|
|
def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
|
|
"""Like pydispatcher.robust.sendRobust but it also logs errors and returns
|
|
Failures instead of exceptions.
|
|
"""
|
|
dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
|
|
spider = named.get('spider', None)
|
|
responses = []
|
|
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
|
try:
|
|
response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named)
|
|
if isinstance(response, Deferred):
|
|
logger.error("Cannot return deferreds from signal handler: %(receiver)s",
|
|
{'receiver': receiver}, extra={'spider': spider})
|
|
except dont_log:
|
|
result = Failure()
|
|
except Exception:
|
|
result = Failure()
|
|
logger.error("Error caught on signal handler: %(receiver)s",
|
|
{'receiver': receiver},
|
|
exc_info=True, extra={'spider': spider})
|
|
else:
|
|
result = response
|
|
responses.append((receiver, result))
|
|
return responses
|
|
|
|
|
|
def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
|
|
"""Like send_catch_log but supports returning deferreds on signal handlers.
|
|
Returns a deferred that gets fired once all signal handlers deferreds were
|
|
fired.
|
|
"""
|
|
def logerror(failure, recv):
|
|
if dont_log is None or not isinstance(failure.value, dont_log):
|
|
logger.error("Error caught on signal handler: %(receiver)s",
|
|
{'receiver': recv},
|
|
exc_info=failure_to_exc_info(failure),
|
|
extra={'spider': spider})
|
|
return failure
|
|
|
|
dont_log = named.pop('dont_log', None)
|
|
spider = named.get('spider', None)
|
|
dfds = []
|
|
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
|
d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named)
|
|
d.addErrback(logerror, receiver)
|
|
d.addBoth(lambda result: (receiver, result))
|
|
dfds.append(d)
|
|
d = DeferredList(dfds)
|
|
d.addCallback(lambda out: [x[1] for x in out])
|
|
return d
|
|
|
|
|
|
def disconnect_all(signal=Any, sender=Any):
|
|
"""Disconnect all signal handlers. Useful for cleaning up after running
|
|
tests
|
|
"""
|
|
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
|
disconnect(receiver, signal=signal, sender=sender)
|