1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
################################################################################
import subprocess, os, sys, re, difflib
################################################################################
IGNORE = (
'.svn',
'infinite_loop',
)
NORMALIZERS = (
(r"Ran (\d+) tests in (\d+\.\d+)s", "Ran \\1 tests in X.XXXs" ),
(r'File ".*?([^/\\.]+\.py)"', 'File "\\1"'),
)
################################################################################
def norm_result(result):
"normalize differences, such as timing between output"
for normalizer, replacement in NORMALIZERS:
if hasattr(normalizer, '__call__'):
result = normalizer(result)
else:
result = re.sub(normalizer, replacement, result)
return result
def call_proc(cmd, cd=None):
proc = subprocess.Popen (
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd = cd,
universal_newlines = True,
)
if proc.wait():
print ("%s %s" % (cmd, proc.wait()))
raise Exception(proc.stdout.read())
return proc.stdout.read()
################################################################################
unnormed_diff = '-u' in sys.argv
verbose = '-v' in sys.argv or unnormed_diff
if '-h' in sys.argv or '--help' in sys.argv: sys.exit (
"\nCOMPARES OUTPUT OF SINGLE VS SUBPROCESS MODE OF RUN_TESTS.PY\n\n"
'-v, to output diffs even on success\n'
'-u, to output diffs of unnormalized tests\n\n'
"Each line of a Differ delta begins with a two-letter code:\n\n"
" '- ' line unique to sequence 1\n"
" '+ ' line unique to sequence 2\n"
" ' ' line common to both sequences\n"
" '? ' line not present in either input sequence\n"
)
main_dir = os.path.split(os.path.abspath(sys.argv[0]))[0]
trunk_dir = os.path.normpath(os.path.join(main_dir, '../../'))
test_suite_dirs = [x for x in os.listdir(main_dir)
if os.path.isdir(os.path.join(main_dir, x))
and x not in IGNORE ]
################################################################################
def assert_on_results(suite, single, sub):
test = globals().get('%s_test' % suite)
if hasattr(test, '__call_'):
test(suite, single, sub)
print ("assertions on %s OK" % (suite,))
# Don't modify tests in suites below. These assertions are in place to make sure
# that tests are actually being ran
def all_ok_test(uite, *args):
for results in args:
assert "Ran 36 tests" in results # some tests are runing
assert "OK" in results # OK
def failures1_test(suite, *args):
for results in args:
assert "FAILED (failures=2)" in results
assert "Ran 18 tests" in results
################################################################################
# Test that output is the same in single process and subprocess modes
#
base_cmd = [sys.executable, 'run_tests.py', '-i']
cmd = base_cmd + ['-n', '-f']
sub_cmd = base_cmd + ['-f']
time_out_cmd = base_cmd + ['-t', '4', '-f', 'infinite_loop' ]
passes = 0
failed = False
for suite in test_suite_dirs:
single = call_proc(cmd + [suite], trunk_dir)
subs = call_proc(sub_cmd + [suite], trunk_dir)
normed_single, normed_subs = map(norm_result,(single, subs))
failed = normed_single != normed_subs
if failed:
print ('%s suite comparison FAILED\n' % (suite,))
else:
passes += 1
print ('%s suite comparison OK' % (suite,))
assert_on_results(suite, single, subs)
if verbose or failed:
print ("difflib.Differ().compare(single, suprocessed):\n")
print (''.join ( list(
difflib.Differ().compare (
(unnormed_diff and single or normed_single).splitlines(1),
(unnormed_diff and subs or normed_subs).splitlines(1)
))
))
sys.stdout.write("infinite_loop suite (subprocess mode timeout) ")
loop_test = call_proc(time_out_cmd, trunk_dir)
assert "successfully terminated" in loop_test
passes += 1
print ("OK")
print ("\n%s/%s suites pass" % (passes, len(test_suite_dirs) + 1))
print ("\n-h for help")
################################################################################
|
