1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
Part of the astor library for Python AST manipulation.
License: 3-clause BSD
Copyright (c) 2015 Patrick Maupin
"""
import sys
import os
import ast
import shutil
import logging
from astor.code_gen import to_source
from astor.file_util import code_to_ast
from astor.node_util import (allow_ast_comparison, dump_tree,
strip_tree, fast_compare)
dsttree = 'tmp_rtrip'
# TODO: Remove this workaround once we remove version 2 support
def out_prep(s, pre_encoded=(sys.version_info[0] == 2)):
return s if pre_encoded else s.encode('utf-8')
def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False,
ignore_exceptions=False, fullcomp=False):
"""Walk the srctree, and convert/copy all python files
into the dsttree
"""
if fullcomp:
allow_ast_comparison()
parse_file = code_to_ast.parse_file
find_py_files = code_to_ast.find_py_files
srctree = os.path.normpath(srctree)
if not readonly:
dsttree = os.path.normpath(dsttree)
logging.info('')
logging.info('Trashing ' + dsttree)
shutil.rmtree(dsttree, True)
unknown_src_nodes = set()
unknown_dst_nodes = set()
badfiles = set()
broken = []
oldpath = None
allfiles = find_py_files(srctree, None if readonly else dsttree)
for srcpath, fname in allfiles:
# Create destination directory
if not readonly and srcpath != oldpath:
oldpath = srcpath
if srcpath >= srctree:
dstpath = srcpath.replace(srctree, dsttree, 1)
if not dstpath.startswith(dsttree):
raise ValueError("%s not a subdirectory of %s" %
(dstpath, dsttree))
else:
assert srctree.startswith(srcpath)
dstpath = dsttree
os.makedirs(dstpath)
srcfname = os.path.join(srcpath, fname)
logging.info('Converting %s' % srcfname)
try:
srcast = parse_file(srcfname)
except SyntaxError:
badfiles.add(srcfname)
continue
try:
dsttxt = to_source(srcast)
except Exception:
if not ignore_exceptions:
raise
dsttxt = ''
if not readonly:
dstfname = os.path.join(dstpath, fname)
try:
with open(dstfname, 'wb') as f:
f.write(out_prep(dsttxt))
except UnicodeEncodeError:
badfiles.add(dstfname)
# As a sanity check, make sure that ASTs themselves
# round-trip OK
try:
dstast = ast.parse(dsttxt) if readonly else parse_file(dstfname)
except SyntaxError:
dstast = []
if fullcomp:
unknown_src_nodes.update(strip_tree(srcast))
unknown_dst_nodes.update(strip_tree(dstast))
bad = srcast != dstast
else:
bad = not fast_compare(srcast, dstast)
if dumpall or bad:
srcdump = dump_tree(srcast)
dstdump = dump_tree(dstast)
logging.warning(' calculating dump -- %s' %
('bad' if bad else 'OK'))
if bad:
broken.append(srcfname)
if dumpall or bad:
if not readonly:
try:
with open(dstfname[:-3] + '.srcdmp', 'wb') as f:
f.write(out_prep(srcdump))
except UnicodeEncodeError:
badfiles.add(dstfname[:-3] + '.srcdmp')
try:
with open(dstfname[:-3] + '.dstdmp', 'wb') as f:
f.write(out_prep(dstdump))
except UnicodeEncodeError:
badfiles.add(dstfname[:-3] + '.dstdmp')
elif dumpall:
sys.stdout.write('\n\nAST:\n\n ')
sys.stdout.write(srcdump.replace('\n', '\n '))
sys.stdout.write('\n\nDecompile:\n\n ')
sys.stdout.write(dsttxt.replace('\n', '\n '))
sys.stdout.write('\n\nNew AST:\n\n ')
sys.stdout.write('(same as old)' if dstdump == srcdump
else dstdump.replace('\n', '\n '))
sys.stdout.write('\n')
if badfiles:
logging.warning('\nFiles not processed due to syntax errors:')
for fname in sorted(badfiles):
logging.warning(' %s' % fname)
if broken:
logging.warning('\nFiles failed to round-trip to AST:')
for srcfname in broken:
logging.warning(' %s' % srcfname)
ok_to_strip = 'col_offset _precedence _use_parens lineno _p_op _pp'
ok_to_strip = set(ok_to_strip.split())
bad_nodes = (unknown_dst_nodes | unknown_src_nodes) - ok_to_strip
if bad_nodes:
logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes)
logging.info('\n')
return broken
def usage(msg):
raise SystemExit(textwrap.dedent("""
Error: %s
Usage:
python -m astor.rtrip [readonly] [<source>]
This utility tests round-tripping of Python source to AST
and back to source.
If readonly is specified, then the source will be tested,
but no files will be written.
if the source is specified to be "stdin" (without quotes)
then any source entered at the command line will be compiled
into an AST, converted back to text, and then compiled to
an AST again, and the results will be displayed to stdout.
If neither readonly nor stdin is specified, then rtrip
will create a mirror directory named tmp_rtrip and will
recursively round-trip all the Python source from the source
into the tmp_rtrip dir, after compiling it and then reconstituting
it through code_gen.to_source.
If the source is not specified, the entire Python library will be used.
""") % msg)
if __name__ == '__main__':
import textwrap
args = sys.argv[1:]
readonly = 'readonly' in args
if readonly:
args.remove('readonly')
if not args:
args = [os.path.dirname(textwrap.__file__)]
if len(args) > 1:
usage("Too many arguments")
fname, = args
dumpall = False
if not os.path.exists(fname):
dumpall = fname == 'stdin' or usage("Cannot find directory %s" % fname)
logging.basicConfig(format='%(msg)s', level=logging.INFO)
convert(fname, readonly=readonly or dumpall, dumpall=dumpall)
|
