Source code for scrapy.extensions.memdebug

"""
MemoryDebugger extension

See documentation in docs/topics/extensions.rst
"""

import gc

from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.trackref import live_refs


[docs]class MemoryDebugger: def __init__(self, stats): self.stats = stats @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o def spider_closed(self, spider, reason): gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider) for cls, wdict in live_refs.items(): if not wdict: continue self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)