1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
from unittest.mock import MagicMock
import pytest
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.extensions import statsmailer
from scrapy.mail import MailSender
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.spider import DefaultSpider
@pytest.fixture
def dummy_stats():
class DummyStats(StatsCollector):
def __init__(self):
# pylint: disable=super-init-not-called
self._stats = {"global_item_scraped_count": 42}
def get_stats(self):
return {"item_scraped_count": 10, **self._stats}
return DummyStats()
def test_from_crawler_without_recipients_raises_notconfigured():
crawler = MagicMock()
crawler.settings.getlist.return_value = []
crawler.stats = MagicMock()
with pytest.raises(NotConfigured):
statsmailer.StatsMailer.from_crawler(crawler)
def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch):
crawler = MagicMock()
crawler.settings.getlist.return_value = ["test@example.com"]
crawler.stats = dummy_stats
crawler.signals = SignalManager(crawler)
mailer = MagicMock(spec=MailSender)
monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
ext = statsmailer.StatsMailer.from_crawler(crawler)
assert isinstance(ext, statsmailer.StatsMailer)
assert ext.recipients == ["test@example.com"]
assert ext.mail is mailer
def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch):
crawler = MagicMock()
crawler.settings.getlist.return_value = ["test@example.com"]
crawler.stats = dummy_stats
crawler.signals = SignalManager(crawler)
mailer = MagicMock(spec=MailSender)
monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
statsmailer.StatsMailer.from_crawler(crawler)
connected = crawler.signals.send_catch_log(
signals.spider_closed, spider=DefaultSpider(name="dummy")
)
assert connected is not None
def test_spider_closed_sends_email(dummy_stats):
recipients = ["test@example.com"]
mail = MagicMock(spec=MailSender)
ext = statsmailer.StatsMailer(dummy_stats, recipients, mail)
spider = DefaultSpider(name="dummy")
ext.spider_closed(spider)
args, _ = mail.send.call_args
to, subject, body = args
assert to == recipients
assert "Scrapy stats for: dummy" in subject
assert "global_item_scraped_count" in body
assert "item_scraped_count" in body
|
