1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
|
#!/usr/bin/env python
"""Test suite for untokenize."""
from __future__ import (absolute_import,
division,
print_function,
unicode_literals)
import io
import sys
import tokenize
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
import untokenize
class TestUnits(unittest.TestCase):
def check(self, source_code):
string_io = io.StringIO(source_code)
self.assertEqual(
source_code,
untokenize.untokenize(
tokenize.generate_tokens(string_io.readline)))
def test_untokenize(self):
self.check('''
def zap():
"""Hello zap.
"""; 1
x \t= \t\t \t 1
''')
def test_untokenize_with_tab_indentation(self):
self.check("""
if True:
\tdef zap():
\t\tx \t= \t\t \t 1
""")
def test_untokenize_with_backslash_in_comment(self):
self.check(r'''
def foo():
"""Hello foo."""
def zap(): bar(1) # \
''')
def test_untokenize_with_escaped_newline(self):
self.check(r'''def foo():
"""Hello foo."""
x = \
1
''')
def test_untokenize_with_empty_string(self):
self.check('')
@unittest.skipIf(sys.version_info < (3, 0),
'We are testing tokenize.ENCODING in Python 3')
def test_untokenize_with_encoding(self):
source = '0'
bytes_io = io.BytesIO(source.encode('us-ascii'))
self.assertEqual(
source,
untokenize.untokenize(tokenize.tokenize(bytes_io.readline)))
if __name__ == '__main__':
unittest.main()
|
