1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
import pytest
from itemloaders.processors import Compose, Identity, Join, MapCompose, TakeFirst
def test_take_first():
proc = TakeFirst()
assert proc([None, "", "hello", "world"]) == "hello"
assert proc([None, "", 0, "hello", "world"]) == 0
def test_identity():
proc = Identity()
assert proc([None, "", "hello", "world"]) == [None, "", "hello", "world"]
def test_join():
proc = Join()
with pytest.raises(TypeError):
proc([None, "", "hello", "world"])
assert proc(["", "hello", "world"]) == " hello world"
assert proc(["hello", "world"]) == "hello world"
assert isinstance(proc(["hello", "world"]), str)
def test_compose():
proc = Compose(lambda v: v[0], str.upper)
assert proc(["hello", "world"]) == "HELLO"
proc = Compose(str.upper)
assert proc(None) is None
proc = Compose(str.upper, stop_on_none=False)
with pytest.raises(
ValueError,
match=r"Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
):
proc(None)
proc = Compose(str.upper, lambda x: x + 1)
with pytest.raises(
ValueError,
match=r"Error in Compose with .* error='TypeError: (can only|unsupported operand)",
):
proc("hello")
def test_mapcompose():
def filter_world(x):
return None if x == "world" else x
proc = MapCompose(filter_world, str.upper)
assert proc(["hello", "world", "this", "is", "scrapy"]) == [
"HELLO",
"THIS",
"IS",
"SCRAPY",
]
proc = MapCompose(filter_world, str.upper)
assert proc(None) == []
proc = MapCompose(filter_world, str.upper)
with pytest.raises(
ValueError,
match=r"Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)",
):
proc([1])
proc = MapCompose(filter_world, lambda x: x + 1)
with pytest.raises(
ValueError,
match=r"Error in MapCompose with .* error='TypeError: (can only|unsupported operand)",
):
proc("hello")
|
