1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
from os.path import join
from twisted.trial import unittest
from twisted.internet import defer
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
from tests import tests_datadir
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
command = 'shell'
@defer.inlineCallbacks
def test_empty(self):
_, out, _ = yield self.execute(['-c', 'item'])
assert b'{}' in out
@defer.inlineCallbacks
def test_response_body(self):
_, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body'])
assert b'Works' in out
@defer.inlineCallbacks
def test_response_type_text(self):
_, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)'])
assert b'TextResponse' in out
@defer.inlineCallbacks
def test_response_type_html(self):
_, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)'])
assert b'HtmlResponse' in out
@defer.inlineCallbacks
def test_response_selector_html(self):
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
self.assertEqual(out.strip(), b'Works')
@defer.inlineCallbacks
def test_response_encoding_gb18030(self):
_, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding'])
self.assertEqual(out.strip(), b'gb18030')
@defer.inlineCallbacks
def test_redirect(self):
_, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url'])
assert out.strip().endswith(b'/redirected')
@defer.inlineCallbacks
def test_redirect_follow_302(self):
_, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
assert out.strip().endswith(b'200')
@defer.inlineCallbacks
def test_redirect_not_follow_302(self):
_, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'])
assert out.strip().endswith(b'302')
@defer.inlineCallbacks
def test_fetch_redirect_follow_302(self):
"""Test that calling `fetch(url)` follows HTTP redirects by default."""
url = self.url('/redirect-no-meta-refresh')
code = "fetch('{0}')"
errcode, out, errout = yield self.execute(['-c', code.format(url)])
self.assertEqual(errcode, 0, out)
assert b'Redirecting (302)' in errout
assert b'Crawled (200)' in errout
@defer.inlineCallbacks
def test_fetch_redirect_not_follow_302(self):
"""Test that calling `fetch(url, redirect=False)` disables automatic redirects."""
url = self.url('/redirect-no-meta-refresh')
code = "fetch('{0}', redirect=False)"
errcode, out, errout = yield self.execute(['-c', code.format(url)])
self.assertEqual(errcode, 0, out)
assert b'Crawled (302)' in errout
@defer.inlineCallbacks
def test_request_replace(self):
url = self.url('/text')
code = "fetch('{0}') or fetch(response.request.replace(method='POST'))"
errcode, out, _ = yield self.execute(['-c', code.format(url)])
self.assertEqual(errcode, 0, out)
@defer.inlineCallbacks
def test_scrapy_import(self):
url = self.url('/text')
code = "fetch(scrapy.Request('{0}'))"
errcode, out, _ = yield self.execute(['-c', code.format(url)])
self.assertEqual(errcode, 0, out)
@defer.inlineCallbacks
def test_local_file(self):
filepath = join(tests_datadir, 'test_site/index.html')
_, out, _ = yield self.execute([filepath, '-c', 'item'])
assert b'{}' in out
@defer.inlineCallbacks
def test_local_nofile(self):
filepath = 'file:///tests/sample_data/test_site/nothinghere.html'
errcode, out, err = yield self.execute([filepath, '-c', 'item'],
check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'No such file or directory', err)
@defer.inlineCallbacks
def test_dns_failures(self):
url = 'www.somedomainthatdoesntexi.st'
errcode, out, err = yield self.execute([url, '-c', 'item'],
check_code=False)
self.assertEqual(errcode, 1, out or err)
self.assertIn(b'DNS lookup failed', err)
|
