1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
from os.path import join, abspath
from tests.test_commands import CommandTest
class CheckCommandTest(CommandTest):
command = 'check'
def setUp(self):
super(CheckCommandTest, self).setUp()
self.spider_name = 'check_spider'
self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py'))
def _write_contract(self, contracts, parse_def):
with open(self.spider, 'w') as file:
file.write(f"""
import scrapy
class CheckSpider(scrapy.Spider):
name = '{self.spider_name}'
start_urls = ['http://example.com']
def parse(self, response, **cb_kwargs):
\"\"\"
@url http://example.com
{contracts}
\"\"\"
{parse_def}
""")
def _test_contract(self, contracts='', parse_def='pass'):
self._write_contract(contracts, parse_def)
p, out, err = self.proc('check')
self.assertNotIn('F', out)
self.assertIn('OK', err)
self.assertEqual(p.returncode, 0)
def test_check_returns_requests_contract(self):
contracts = """
@returns requests 1
"""
parse_def = """
yield scrapy.Request(url='http://next-url.com')
"""
self._test_contract(contracts, parse_def)
def test_check_returns_items_contract(self):
contracts = """
@returns items 1
"""
parse_def = """
yield {'key1': 'val1', 'key2': 'val2'}
"""
self._test_contract(contracts, parse_def)
def test_check_cb_kwargs_contract(self):
contracts = """
@cb_kwargs {"arg1": "val1", "arg2": "val2"}
"""
parse_def = """
if len(cb_kwargs.items()) == 0:
raise Exception("Callback args not set")
"""
self._test_contract(contracts, parse_def)
def test_check_scrapes_contract(self):
contracts = """
@scrapes key1 key2
"""
parse_def = """
yield {'key1': 'val1', 'key2': 'val2'}
"""
self._test_contract(contracts, parse_def)
def test_check_all_default_contracts(self):
contracts = """
@returns items 1
@returns requests 1
@scrapes key1 key2
@cb_kwargs {"arg1": "val1", "arg2": "val2"}
"""
parse_def = """
yield {'key1': 'val1', 'key2': 'val2'}
yield scrapy.Request(url='http://next-url.com')
if len(cb_kwargs.items()) == 0:
raise Exception("Callback args not set")
"""
self._test_contract(contracts, parse_def)
def test_SCRAPY_CHECK_set(self):
parse_def = """
import os
if not os.environ.get('SCRAPY_CHECK'):
raise Exception('SCRAPY_CHECK not set')
"""
self._test_contract(parse_def=parse_def)
|
