1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
module Rugments
module Lexers
class Python < RegexLexer
title 'Python'
desc 'The Python programming language (python.org)'
tag 'python'
aliases 'py'
filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'
mimetypes 'text/x-python', 'application/x-python'
def self.analyze_text(text)
return 1 if text.shebang?(/pythonw?(3|2(\.\d)?)?/)
end
def self.keywords
@keywords ||= %w(
assert break continue del elif else except exec
finally for global if lambda pass print raise
return try while yield as with
)
end
def self.builtins
@builtins ||= %w(
__import__ abs all any apply basestring bin bool buffer
bytearray bytes callable chr classmethod cmp coerce compile
complex delattr dict dir divmod enumerate eval execfile exit
file filter float frozenset getattr globals hasattr hash hex id
input int intern isinstance issubclass iter len list locals
long map max min next object oct open ord pow property range
raw_input reduce reload repr reversed round set setattr slice
sorted staticmethod str sum super tuple type unichr unicode
vars xrange zip
)
end
def self.builtins_pseudo
@builtins_pseudo ||= %w(self None Ellipsis NotImplemented False True)
end
def self.exceptions
@exceptions ||= %w(
ArithmeticError AssertionError AttributeError
BaseException DeprecationWarning EOFError EnvironmentError
Exception FloatingPointError FutureWarning GeneratorExit IOError
ImportError ImportWarning IndentationError IndexError KeyError
KeyboardInterrupt LookupError MemoryError NameError
NotImplemented NotImplementedError OSError OverflowError
OverflowWarning PendingDeprecationWarning ReferenceError
RuntimeError RuntimeWarning StandardError StopIteration
SyntaxError SyntaxWarning SystemError SystemExit TabError
TypeError UnboundLocalError UnicodeDecodeError
UnicodeEncodeError UnicodeError UnicodeTranslateError
UnicodeWarning UserWarning ValueError VMSError Warning
WindowsError ZeroDivisionError
)
end
identifier = /[a-z_][a-z0-9_]*/i
dotted_identifier = /[a-z_.][a-z0-9_.]*/i
state :root do
rule /\n+/m, Text
rule /^(:)(\s*)([ru]{,2}""".*?""")/mi do
groups Punctuation, Text, Str::Doc
end
rule /[^\S\n]+/, Text
rule /#.*$/, Comment
rule /[\[\]{}:(),;]/, Punctuation
rule /\\\n/, Text
rule /\\/, Text
rule /(in|is|and|or|not)\b/, Operator::Word
rule /!=|==|<<|>>|[-~+\/*%=<>&^|.]/, Operator
rule /(def)((?:\s|\\\s)+)/ do
groups Keyword, Text
push :funcname
end
rule /(class)((?:\s|\\\s)+)/ do
groups Keyword, Text
push :classname
end
rule /(from)((?:\s|\\\s)+)/ do
groups Keyword::Namespace, Text
push :fromimport
end
rule /(import)((?:\s|\\\s)+)/ do
groups Keyword::Namespace, Text
push :import
end
# TODO: not in python 3
rule /`.*?`/, Str::Backtick
rule /(?:r|ur|ru)"""/i, Str, :tdqs
rule /(?:r|ur|ru)'''/i, Str, :tsqs
rule /(?:r|ur|ru)"/i, Str, :dqs
rule /(?:r|ur|ru)'/i, Str, :sqs
rule /u?"""/i, Str, :escape_tdqs
rule /u?'''/i, Str, :escape_tsqs
rule /u?"/i, Str, :escape_dqs
rule /u?'/i, Str, :escape_sqs
rule /@#{dotted_identifier}/i, Name::Decorator
# using negative lookbehind so we don't match property names
rule /(?<!\.)#{identifier}/ do |m|
if self.class.keywords.include? m[0]
token Keyword
elsif self.class.exceptions.include? m[0]
token Name::Builtin
elsif self.class.builtins.include? m[0]
token Name::Builtin
elsif self.class.builtins_pseudo.include? m[0]
token Name::Builtin::Pseudo
else
token Name
end
end
rule identifier, Name
rule /(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?/i, Num::Float
rule /\d+e[+-]?[0-9]+/i, Num::Float
rule /0[0-7]+/, Num::Oct
rule /0x[a-f0-9]+/i, Num::Hex
rule /\d+L/, Num::Integer::Long
rule /\d+/, Num::Integer
end
state :funcname do
rule identifier, Name::Function, :pop!
end
state :classname do
rule identifier, Name::Class, :pop!
end
state :import do
# non-line-terminating whitespace
rule /(?:[ \t]|\\\n)+/, Text
rule /as\b/, Keyword::Namespace
rule /,/, Operator
rule dotted_identifier, Name::Namespace
rule(//) { pop! } # anything else -> go back
end
state :fromimport do
# non-line-terminating whitespace
rule /(?:[ \t]|\\\n)+/, Text
rule /import\b/, Keyword::Namespace, :pop!
rule dotted_identifier, Name::Namespace
end
state :strings do
rule /%(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str::Interpol
end
state :strings_double do
rule /[^\\"%\n]+/, Str
mixin :strings
end
state :strings_single do
rule /[^\\'%\n]+/, Str
mixin :strings
end
state :nl do
rule /\n/, Str
end
state :escape do
rule %r(\\
( [\\abfnrtv"']
| \n
| N{.*?}
| u[a-fA-F0-9]{4}
| U[a-fA-F0-9]{8}
| x[a-fA-F0-9]{2}
| [0-7]{1,3}
)
)x, Str::Escape
end
state :dqs do
rule /"/, Str, :pop!
rule /\\\\|\\"|\\\n/, Str::Escape
mixin :strings_double
end
state :sqs do
rule /'/, Str, :pop!
rule /\\\\|\\'|\\\n/, Str::Escape
mixin :strings_single
end
state :tdqs do
rule /"""/, Str, :pop!
rule /"/, Str
mixin :strings_double
mixin :nl
end
state :tsqs do
rule /'''/, Str, :pop!
rule /'/, Str
mixin :strings_single
mixin :nl
end
%w(tdqs tsqs dqs sqs).each do |qtype|
state :"escape_#{qtype}" do
mixin :escape
mixin :"#{qtype}"
end
end
end
end
end
|
