1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
"""
MemoryDebugger extension
See documentation in docs/topics/extensions.rst
"""
import gc
import socket
import libxml2
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core import signals
from scrapy.core.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.conf import settings
from scrapy import log
class MemoryDebugger(object):
def __init__(self):
if not settings.getbool('MEMDEBUG_ENABLED'):
raise NotConfigured
self.mail = MailSender()
self.rcpts = settings.getlist('MEMDEBUG_NOTIFY')
dispatcher.connect(self.engine_started, signals.engine_started)
dispatcher.connect(self.engine_stopped, signals.engine_stopped)
def engine_started(self):
libxml2.debugMemory(1)
def engine_stopped(self):
figures = self.collect_figures()
report = self.create_report(figures)
self.log_or_send_report(report)
def collect_figures(self):
libxml2.cleanupParser()
gc.collect()
figures = []
figures.append(("Objects in gc.garbage", len(gc.garbage), ""))
figures.append(("libxml2 memory leak", libxml2.debugMemory(1), "bytes"))
return figures
def create_report(self, figures):
s = ""
s += "SCRAPY MEMORY DEBUGGER RESULTS\n\n"
for f in figures:
s += "%-30s : %d %s\n" % f
return s
def log_or_send_report(self, report):
if self.rcpts:
self.mail.send(self.rcpts, "Scrapy Memory Debugger results at %s" % \
socket.gethostname(), report)
log.msg(report)
|
