1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
|
# Copyright (c) 2014, Menno Smits
# Released subject to the New BSD License
# Please see http://en.wikipedia.org/wiki/BSD_licenses
import unittest
from imapclient.response_lexer import TokenSource
class TestTokenSource(unittest.TestCase):
def test_one_token(self):
self.check([b"abc"], [b"abc"])
def test_simple_tokens(self):
self.check([b"abc 111 def"], [b"abc", b"111", b"def"])
def test_multiple_inputs(self):
self.check([b"abc 111", b"def 222"], [b"abc", b"111", b"def", b"222"])
def test_whitespace(self):
self.check([b"abc def"], [b"abc", b"def"])
self.check([b" abc \t\t\r\r\n\n def "], [b"abc", b"def"])
def test_quoted_strings(self):
self.check([b'"abc def"'], [b'"abc def"'])
self.check([b'""'], [b'""'])
self.check([b'111 "abc def" 222'], [b"111", b'"abc def"', b"222"])
def test_unterminated_strings(self):
message = "No closing '\"'"
self.check_error([b'"'], message)
self.check_error([b'"aaa bbb'], message)
def test_escaping(self):
self.check([rb'"aaa\"bbb"'], [rb'"aaa"bbb"'])
self.check([rb'"aaa\\bbb"'], [rb'"aaa\bbb"'])
self.check([rb'"aaa\\bbb \"\""'], [rb'"aaa\bbb """'])
def test_invalid_escape(self):
self.check([rb'"aaa\Zbbb"'], [rb'"aaa\Zbbb"'])
def test_lists(self):
self.check([b"()"], [b"(", b")"])
self.check([b"(aaa)"], [b"(", b"aaa", b")"])
self.check(
[b'(aaa "bbb def" 123)'], [b"(", b"aaa", b'"bbb def"', b"123", b")"]
)
self.check(
[b"(aaa)(bbb ccc)"], [b"(", b"aaa", b")", b"(", b"bbb", b"ccc", b")"]
)
self.check(
[b"(aaa (bbb ccc))"], [b"(", b"aaa", b"(", b"bbb", b"ccc", b")", b")"]
)
def test_square_brackets(self):
self.check([b"[aaa bbb]"], [b"[aaa bbb]"])
self.check([b"aaa[bbb]"], [b"aaa[bbb]"])
self.check([b"[bbb]aaa"], [b"[bbb]aaa"])
self.check([b"aaa [bbb]"], [b"aaa", b"[bbb]"])
def test_no_escaping_in_square_brackets(self):
self.check([rb"[aaa\\bbb]"], [rb"[aaa\\bbb]"])
def test_unmatched_square_brackets(self):
message = "No closing ']'"
self.check_error([b"["], message)
self.check_error([b"[aaa bbb"], message)
def test_literal(self):
source = TokenSource([(b"abc {7}", b"foo bar"), b")"])
tokens = iter(source)
self.assertEqual(next(tokens), b"abc")
self.assertEqual(next(tokens), b"{7}")
self.assertEqual(source.current_literal, b"foo bar")
self.assertEqual(next(tokens), b")")
self.assertRaises(StopIteration, lambda: next(tokens))
def test_literals(self):
source = TokenSource([(b"abc {7}", b"foo bar"), (b"{5}", b"snafu"), b")"])
tokens = iter(source)
self.assertEqual(next(tokens), b"abc")
self.assertEqual(next(tokens), b"{7}")
self.assertEqual(source.current_literal, b"foo bar")
self.assertEqual(next(tokens), b"{5}")
self.assertEqual(source.current_literal, b"snafu")
self.assertEqual(next(tokens), b")")
self.assertRaises(StopIteration, lambda: next(tokens))
def check(self, text_in, expected_out):
tokens = TokenSource(text_in)
self.assertSequenceEqual(list(tokens), expected_out)
def check_error(self, text_in, expected_message):
self.assertRaisesRegex(
ValueError, expected_message, lambda: list(TokenSource(text_in))
)
|
