1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
import functools
import operator
import unittest
from itertools import count
from scrapy.utils.python import str_to_unicode, unicode_to_str, \
memoizemethod_noargs, isbinarytext, equal_attributes, \
WeakKeyCache, stringify_dict, get_func_args
__doctests__ = ['scrapy.utils.python']
class UtilsPythonTestCase(unittest.TestCase):
def test_str_to_unicode(self):
# converting an utf-8 encoded string to unicode
self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
# converting a latin-1 encoded string to unicode
self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
# converting a unicode to unicode should return the same object
self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
# converting a strange object should raise TypeError
self.assertRaises(TypeError, str_to_unicode, 423)
# check errors argument works
assert u'\ufffd' in str_to_unicode('a\xedb', 'utf-8', errors='replace')
def test_unicode_to_str(self):
# converting a unicode object to an utf-8 encoded string
self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49')
# converting a unicode object to a latin-1 encoded string
self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49')
# converting a regular string to string should return the same object
self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e')
# converting a strange object should raise TypeError
self.assertRaises(TypeError, unicode_to_str, unittest)
# check errors argument works
assert '?' in unicode_to_str(u'a\ufffdb', 'latin-1', errors='replace')
def test_memoizemethod_noargs(self):
class A(object):
@memoizemethod_noargs
def cached(self):
return object()
def noncached(self):
return object()
a = A()
one = a.cached()
two = a.cached()
three = a.noncached()
assert one is two
assert one is not three
def test_isbinarytext(self):
# basic tests
assert not isbinarytext("hello")
# utf-16 strings contain null bytes
assert not isbinarytext(u"hello".encode('utf-16'))
# one with encoding
assert not isbinarytext("<div>Price \xa3</div>")
# finally some real binary bytes
assert isbinarytext("\x02\xa3")
def test_equal_attributes(self):
class Obj:
pass
a = Obj()
b = Obj()
# no attributes given return False
self.failIf(equal_attributes(a, b, []))
# not existent attributes
self.failIf(equal_attributes(a, b, ['x', 'y']))
a.x = 1
b.x = 1
# equal attribute
self.failUnless(equal_attributes(a, b, ['x']))
b.y = 2
# obj1 has no attribute y
self.failIf(equal_attributes(a, b, ['x', 'y']))
a.y = 2
# equal attributes
self.failUnless(equal_attributes(a, b, ['x', 'y']))
a.y = 1
# differente attributes
self.failIf(equal_attributes(a, b, ['x', 'y']))
# test callable
a.meta = {}
b.meta = {}
self.failUnless(equal_attributes(a, b, ['meta']))
# compare ['meta']['a']
a.meta['z'] = 1
b.meta['z'] = 1
get_z = operator.itemgetter('z')
get_meta = operator.attrgetter('meta')
compare_z = lambda obj: get_z(get_meta(obj))
self.failUnless(equal_attributes(a, b, [compare_z, 'x']))
# fail z equality
a.meta['z'] = 2
self.failIf(equal_attributes(a, b, [compare_z, 'x']))
def test_weakkeycache(self):
class _Weakme(object): pass
_values = count()
wk = WeakKeyCache(lambda k: next(_values))
k = _Weakme()
v = wk[k]
self.assertEqual(v, wk[k])
self.assertNotEqual(v, wk[_Weakme()])
self.assertEqual(v, wk[k])
del k
self.assertFalse(len(wk._weakdict))
def test_stringify_dict(self):
d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
d2 = stringify_dict(d, keys_only=False)
self.failUnlessEqual(d, d2)
self.failIf(d is d2) # shouldn't modify in place
self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
self.failIf(any(isinstance(x, unicode) for x in d2.values()))
def test_stringify_dict_tuples(self):
tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
d = dict(tuples)
d2 = stringify_dict(tuples, keys_only=False)
self.failUnlessEqual(d, d2)
self.failIf(d is d2) # shouldn't modify in place
self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
self.failIf(any(isinstance(x, unicode) for x in d2.values()))
def test_stringify_dict_keys_only(self):
d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
d2 = stringify_dict(d)
self.failUnlessEqual(d, d2)
self.failIf(d is d2) # shouldn't modify in place
self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
def test_get_func_args(self):
def f1(a, b, c):
pass
def f2(a, b=None, c=None):
pass
class A(object):
def __init__(self, a, b, c):
pass
def method(self, a, b, c):
pass
class Callable(object):
def __call__(self, a, b, c):
pass
a = A(1, 2, 3)
cal = Callable()
partial_f1 = functools.partial(f1, None)
partial_f2 = functools.partial(f1, b=None)
partial_f3 = functools.partial(partial_f2, None)
self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
self.assertEqual(get_func_args(partial_f1), ['b', 'c'])
self.assertEqual(get_func_args(partial_f2), ['a', 'c'])
self.assertEqual(get_func_args(partial_f3), ['c'])
self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
self.assertEqual(get_func_args(object), [])
# TODO: how do we fix this to return the actual argument names?
self.assertEqual(get_func_args(unicode.split), [])
self.assertEqual(get_func_args(" ".join), [])
if __name__ == "__main__":
unittest.main()
|
