1"""
2MemoryDebugger extension
3
4See documentation in docs/topics/extensions.rst
5"""
6
7import gc
8
9from scrapy import signals
10from scrapy.exceptions import NotConfigured
11from scrapy.utils.trackref import live_refs
12
13
14class MemoryDebugger:
15
16    def __init__(self, stats):
17        self.stats = stats
18
19    @classmethod
20    def from_crawler(cls, crawler):
21        if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
22            raise NotConfigured
23        o = cls(crawler.stats)
24        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
25        return o
26
27    def spider_closed(self, spider, reason):
28        gc.collect()
29        self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
30        for cls, wdict in live_refs.items():
31            if not wdict:
32                continue
33            self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider)
34