1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
# benchmark -- automated system for testing distcc correctness
# and performance on various source trees.
# Copyright (C) 2002, 2003 by Martin Pool
# Copyright 2008 Google Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
import commands
import os
import shutil
import stat
import sys
import tempfile
import buildutil
STANDARD_CC_NAMES = ['cc', 'gcc']
STANDARD_CXX_NAMES = ['cxx', 'c++', 'g++' ]
def _find_executable(name):
(rs, output) = commands.getstatusoutput('which "%s"' % name)
if rs:
sys.exit("Could not determine location of '%s'" % name)
return output.strip()
class CompilerSpec:
"""Describes a compiler/make setup.
Used to define different situations such as local compilation, and
various degrees of parallelism."""
def __init__(self, where, cc, cxx, prefix='', make_opts='',
pump_cmd='', num_hosts=1, host_opts='',
name=None):
"""Constructor:
Args:
where: 'local', 'dist', 'lzo', or 'pump'
cc: location of the C compiler
cxx: location of the C++
prefix: a string, either 'distcc ' or ''
make_opts: options to make, such as '-j120'
host_opts: for appending to hosts in DISTCC_HOSTS
such as ',lzo,cpp'
name: a string
"""
self.where = where
self.real_cc = _find_executable(cc)
self.real_cxx = _find_executable(cxx)
self.cc = prefix + self.real_cc
self.cxx = prefix + self.real_cxx
self.make_opts = make_opts
self.host_opts = host_opts
self.pump_cmd = pump_cmd
self.num_hosts = num_hosts
self.host_opts = host_opts
self.name = name or (self.pump_cmd + self.real_cc + "__" +
self.make_opts).replace(' ', '_')
def prepare_shell_script_farm(self, farm_dir, masquerade):
"""Prepare farm directory for masquerading.
Assume the compiler is not local. Each standard name, such as
'cc', is used for form a shell script, named 'cc', that
contains the line 'distcc /my/path/gcc "$@"', where
'/my/path/gcc' is the value of the compiler.gcc field.
If the compiler is local, then the same procedure is followed
except that 'distcc' is omitted from the command line.
"""
assert os.path.isdir(farm_dir)
assert os.path.isabs(farm_dir)
def make_shell_script(name, compiler_path, where):
fd = open(os.path.join(farm_dir, name), 'w')
fd.write('#!/bin/sh\n%s%s "$@"'
% (where != 'local' and 'distcc ' or '',
compiler_path))
fd.close()
os.chmod(os.path.join(farm_dir, name),
stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
for generic_name in STANDARD_CC_NAMES:
make_shell_script(generic_name, self.real_cc, self.where)
for generic_name in STANDARD_CXX_NAMES:
make_shell_script(generic_name, self.real_cxx, self.where)
# Make shell wrapper to help manual debugging.
fd = open(masquerade, 'w')
fd.write("""\
#!/bin/sh
# Execute $@, but force 'cc' and 'cxx'" to be those in the farm of
# masquerading scripts. Each script in turn executes 'distcc' with the actual
# compiler specified with the benchmark.py command.
PATH=%s:"$PATH" "$@"
""" % farm_dir)
fd.close()
os.chmod(masquerade,
stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
def default_compilers(cc, cxx):
return [parse_compiler_opt('local,h1,j1', cc, cxx),
parse_compiler_opt('dist,h10,j20', cc, cxx),
parse_compiler_opt('dist,h10,j40', cc, cxx),
parse_compiler_opt('pump,h10,j20', cc, cxx),
parse_compiler_opt('pump,h10,j40', cc, cxx),
]
def parse_compiler_opt(optarg, cc, cxx):
"""Parse command-line specification of a compiler (-c/--compiler).
XXX: I don't really know what the best syntax for this is. For
the moment, it is "local", "dist", "lzo", or "pump", followed by ",h"
and the number of hosts to use, followed by ",j" and the number
of jobs to use (for the -j option to make).
"""
where, hosts, jobs = optarg.split(',')
if hosts.startswith("h"):
hosts = int(hosts[1:])
if not os.getenv("DISTCC_HOSTS"):
raise ValueError, "You must set DISTCC_HOSTS before running benchmarks"
max_hosts = buildutil.count_hosts(os.getenv("DISTCC_HOSTS"))
if hosts > max_hosts:
print ("Warning: can't use %d hosts: DISTCC_HOSTS only has %d" %
(hosts, max_hosts))
hosts = max_hosts
else:
raise ValueError, ("invalid compiler option: "
"expecting '...,h<NUMBER OF HOSTS>,...', found %s"
% `hosts`)
if jobs.startswith("j"):
jobs = int(jobs[1:])
else:
raise ValueError, ("invalid compiler option: "
"expecting '...,j<NUMBER OF JOBS>', found %s"
% `jobs`)
if where == 'local':
return CompilerSpec(where=where,
name='local_%02d' % jobs,
cc=cc,
cxx=cxx,
num_hosts=1,
make_opts='-j%d' % jobs)
elif where == 'dist':
return CompilerSpec(where=where,
name='dist_h%02d_j%02d' % (hosts, jobs),
cc=cc,
cxx=cxx,
prefix='distcc ',
num_hosts=hosts,
make_opts='-j%d' % jobs)
elif where == 'lzo':
return CompilerSpec(where=where,
name='lzo_h%02d_j%02d' % (hosts, jobs),
cc=cc,
cxx=cxx,
prefix='distcc ',
num_hosts=hosts,
host_opts=",lzo",
make_opts='-j%d' % jobs)
elif where == 'pump':
return CompilerSpec(where=where,
name='pump_h%02d_j%02d' % (hosts, jobs),
cc=cc,
cxx=cxx,
prefix='distcc ',
pump_cmd='pump ',
num_hosts=hosts,
host_opts=",cpp,lzo",
make_opts='-j%d' % jobs)
else:
raise ValueError, ("invalid compiler option: don't understand %s"
% `where`)
def prepare_shell_script_farm(compiler, farm_dir, masquerade):
compiler.prepare_shell_script_farm(farm_dir, masquerade)
|
