1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
#!/usr/bin/python
# -*- coding: utf-8 -*-
import cgi, string, sys, cStringIO
import keyword, token, tokenize
_KEYWORD = token.NT_OFFSET + 1
_TEXT = token.NT_OFFSET + 2
_PROMPT = token.NT_OFFSET + 3
_WHITESPACE = token.NT_OFFSET + 3
import token, tokenize
class TokenFormatter(object):
def __init__(self):
self.css = {
token.NUMBER: 'num',
tokenize.OP: 'op',
token.STRING: 'str',
tokenize.COMMENT: 'com',
tokenize.NAME: 'name',
tokenize.ERRORTOKEN: 'err',
_KEYWORD: 'kwd',
_TEXT: 'txt',
_PROMPT: 'pmt',
}
self.indent_level = 0
self.indent_text = " " * 4
self.prompt_text = '>>>'
def new_line(self):
return "<br />" + self.indent_level * self.indent_text
def classify(self, toktype, toktext):
if toktype in self.css:
s = '<span class="%s">%s</span>' % (self.css[toktype], cgi.escape(toktext).replace(' ', ' '))
else:
s = ''
return s
def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
if toktype == tokenize.INDENT:
self.indent_level += 1
return self.indent_text
elif toktype == tokenize.DEDENT:
self.indent_level -= 1
return ''
elif toktype == _WHITESPACE:
return toktext.replace(' ', ' ')
elif toktype in (tokenize.NEWLINE, tokenize.NL):
return self.new_line()
else:
if token.LPAR <= toktype and toktype <= token.OP:
toktype = token.OP
if srow == 0 and toktext == self.prompt_text:
toktype = _PROMPT
elif toktype == token.NAME and keyword.iskeyword(toktext):
toktype = _KEYWORD
return self.classify(toktype, toktext)
class Parser(object):
""" Send colored python source.
"""
def __init__(self, raw):
""" Store the source text.
"""
self.raw = raw.strip().expandtabs()
self.format_token = TokenFormatter()
self.out = cStringIO.StringIO()
def format(self):
input = cStringIO.StringIO(self.raw)
self.pos = 0
self.lines = [0, 0]
pos = 0
while True:
try:
pos = self.raw.index('\n', pos) + 1
except ValueError:
break
self.lines.append(pos)
self.lines.append(len(self.raw))
self.out.write('<div class="code">')
try:
tokenize.tokenize(input.readline, self)
except tokenize.TokenError, ex:
msg = ex[0]
line = ex[1][0]
self.out.write("<h3>ERROR: %s</h3>%s\n" % (msg, line))
self.out.write('\n</div>')
return self.out.getvalue()
def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
""" Token handler.
"""
if 0:
print "type", toktype, token.tok_name[toktype], "text", toktext,
print "start", srow,scol, "end", erow,ecol
oldpos = self.pos
newpos = self.lines[srow] + scol
self.pos = newpos + len(toktext)
if newpos > oldpos:
self.out.write(self.format_token(_WHITESPACE, self.raw[oldpos:newpos], (srow,scol), (erow,ecol), line))
self.out.write(self.format_token(toktype, toktext, (srow,scol), (erow,ecol), line))
|
