126 lines
4.9 KiB
Python
126 lines
4.9 KiB
Python
"""
|
|
MemoryUsage extension
|
|
|
|
See documentation in docs/topics/extensions.rst
|
|
"""
|
|
import sys
|
|
import socket
|
|
import logging
|
|
from pprint import pformat
|
|
from importlib import import_module
|
|
|
|
from twisted.internet import task
|
|
|
|
from scrapy import signals
|
|
from scrapy.exceptions import NotConfigured
|
|
from scrapy.mail import MailSender
|
|
from scrapy.utils.engine import get_engine_status
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MemoryUsage:
|
|
|
|
def __init__(self, crawler):
|
|
if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
|
|
raise NotConfigured
|
|
try:
|
|
# stdlib's resource module is only available on unix platforms.
|
|
self.resource = import_module('resource')
|
|
except ImportError:
|
|
raise NotConfigured
|
|
|
|
self.crawler = crawler
|
|
self.warned = False
|
|
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
|
|
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
|
|
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
|
|
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
|
|
self.mail = MailSender.from_settings(crawler.settings)
|
|
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
|
|
crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
|
|
|
|
@classmethod
|
|
def from_crawler(cls, crawler):
|
|
return cls(crawler)
|
|
|
|
def get_virtual_size(self):
|
|
size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
|
|
if sys.platform != 'darwin':
|
|
# on macOS ru_maxrss is in bytes, on Linux it is in KB
|
|
size *= 1024
|
|
return size
|
|
|
|
def engine_started(self):
|
|
self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())
|
|
self.tasks = []
|
|
tsk = task.LoopingCall(self.update)
|
|
self.tasks.append(tsk)
|
|
tsk.start(self.check_interval, now=True)
|
|
if self.limit:
|
|
tsk = task.LoopingCall(self._check_limit)
|
|
self.tasks.append(tsk)
|
|
tsk.start(self.check_interval, now=True)
|
|
if self.warning:
|
|
tsk = task.LoopingCall(self._check_warning)
|
|
self.tasks.append(tsk)
|
|
tsk.start(self.check_interval, now=True)
|
|
|
|
def engine_stopped(self):
|
|
for tsk in self.tasks:
|
|
if tsk.running:
|
|
tsk.stop()
|
|
|
|
def update(self):
|
|
self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
|
|
|
|
def _check_limit(self):
|
|
if self.get_virtual_size() > self.limit:
|
|
self.crawler.stats.set_value('memusage/limit_reached', 1)
|
|
mem = self.limit/1024/1024
|
|
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
|
|
{'memusage': mem}, extra={'crawler': self.crawler})
|
|
if self.notify_mails:
|
|
subj = (
|
|
f"{self.crawler.settings['BOT_NAME']} terminated: "
|
|
f"memory usage exceeded {mem}M at {socket.gethostname()}"
|
|
)
|
|
self._send_report(self.notify_mails, subj)
|
|
self.crawler.stats.set_value('memusage/limit_notified', 1)
|
|
|
|
open_spiders = self.crawler.engine.open_spiders
|
|
if open_spiders:
|
|
for spider in open_spiders:
|
|
self.crawler.engine.close_spider(spider, 'memusage_exceeded')
|
|
else:
|
|
self.crawler.stop()
|
|
|
|
def _check_warning(self):
|
|
if self.warned: # warn only once
|
|
return
|
|
if self.get_virtual_size() > self.warning:
|
|
self.crawler.stats.set_value('memusage/warning_reached', 1)
|
|
mem = self.warning/1024/1024
|
|
logger.warning("Memory usage reached %(memusage)dM",
|
|
{'memusage': mem}, extra={'crawler': self.crawler})
|
|
if self.notify_mails:
|
|
subj = (
|
|
f"{self.crawler.settings['BOT_NAME']} warning: "
|
|
f"memory usage reached {mem}M at {socket.gethostname()}"
|
|
)
|
|
self._send_report(self.notify_mails, subj)
|
|
self.crawler.stats.set_value('memusage/warning_notified', 1)
|
|
self.warned = True
|
|
|
|
def _send_report(self, rcpts, subject):
|
|
"""send notification mail with some additional useful info"""
|
|
stats = self.crawler.stats
|
|
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
|
|
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
|
|
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
|
|
|
|
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
|
|
s += "\r\n"
|
|
s += pformat(get_engine_status(self.crawler.engine))
|
|
s += "\r\n"
|
|
self.mail.send(rcpts, subject, s)
|