1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
"""Test lexers module"""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
from unittest import TestCase
from pygments.token import Token
from pygments.lexers import BashLexer
from .. import lexers
class TestLexers(TestCase):
"""Collection of lexers tests"""
def setUp(self):
self.lexer = lexers.IPythonLexer()
self.bash_lexer = BashLexer()
def testIPythonLexer(self):
fragment = '!echo $HOME\n'
tokens = [
(Token.Operator, '!'),
]
tokens.extend(self.bash_lexer.get_tokens(fragment[1:]))
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
fragment_2 = '!' + fragment
tokens_2 = [
(Token.Operator, '!!'),
] + tokens[1:]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = '\t %%!\n' + fragment[1:]
tokens_2 = [
(Token.Text, '\t '),
(Token.Operator, '%%!'),
(Token.Text, '\n'),
] + tokens[1:]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = 'x = ' + fragment
tokens_2 = [
(Token.Name, 'x'),
(Token.Text, ' '),
(Token.Operator, '='),
(Token.Text, ' '),
] + tokens
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = 'x, = ' + fragment
tokens_2 = [
(Token.Name, 'x'),
(Token.Punctuation, ','),
(Token.Text, ' '),
(Token.Operator, '='),
(Token.Text, ' '),
] + tokens
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = 'x, = %sx ' + fragment[1:]
tokens_2 = [
(Token.Name, 'x'),
(Token.Punctuation, ','),
(Token.Text, ' '),
(Token.Operator, '='),
(Token.Text, ' '),
(Token.Operator, '%'),
(Token.Keyword, 'sx'),
(Token.Text, ' '),
] + tokens[1:]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = 'f = %R function () {}\n'
tokens_2 = [
(Token.Name, 'f'),
(Token.Text, ' '),
(Token.Operator, '='),
(Token.Text, ' '),
(Token.Operator, '%'),
(Token.Keyword, 'R'),
(Token.Text, ' function () {}\n'),
]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = '\t%%xyz\n$foo\n'
tokens_2 = [
(Token.Text, '\t'),
(Token.Operator, '%%'),
(Token.Keyword, 'xyz'),
(Token.Text, '\n$foo\n'),
]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = '%system?\n'
tokens_2 = [
(Token.Operator, '%'),
(Token.Keyword, 'system'),
(Token.Operator, '?'),
(Token.Text, '\n'),
]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = 'x != y\n'
tokens_2 = [
(Token.Name, 'x'),
(Token.Text, ' '),
(Token.Operator, '!='),
(Token.Text, ' '),
(Token.Name, 'y'),
(Token.Text, '\n'),
]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment_2 = ' ?math.sin\n'
tokens_2 = [
(Token.Text, ' '),
(Token.Operator, '?'),
(Token.Text, 'math.sin'),
(Token.Text, '\n'),
]
self.assertEqual(tokens_2, list(self.lexer.get_tokens(fragment_2)))
fragment = ' *int*?\n'
tokens = [
(Token.Text, ' *int*'),
(Token.Operator, '?'),
(Token.Text, '\n'),
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
|
