1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
#!/usr/bin/env python3
# encoding: utf-8
# Thomas Nagy, 2006-2015 (ita)
"""
Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
cc -c ../file1.c ../file2.c ../file3.c
Files are output on the directory where the compiler is called, and dependencies are more difficult
to track (do not run the command on all source files if only one file changes)
As such, we do as if the files were compiled one by one, but no command is actually run:
replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
signatures from each slave and finds out the command-line to run.
Just import this module to start using it:
def build(bld):
bld.load('batched_cc')
Note that this is provided as an example, unity builds are recommended
for best performance results (fewer tasks and fewer jobs to execute).
See waflib/extras/unity.py.
"""
from waflib import Task, Utils
from waflib.TaskGen import extension, feature, after_method
from waflib.Tools import c, cxx
MAX_BATCH = 50
c_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
c_fun, _ = Task.compile_fun_noshell(c_str)
cxx_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
cxx_fun, _ = Task.compile_fun_noshell(cxx_str)
count = 70000
class batch(Task.Task):
color = 'PINK'
after = ['c', 'cxx']
before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']
def uid(self):
return Utils.h_list([Task.Task.uid(self), self.generator.idx, self.generator.path.abspath(), self.generator.target])
def __str__(self):
return 'Batch compilation for %d slaves' % len(self.slaves)
def __init__(self, *k, **kw):
Task.Task.__init__(self, *k, **kw)
self.slaves = []
self.inputs = []
self.hasrun = 0
global count
count += 1
self.idx = count
def add_slave(self, slave):
self.slaves.append(slave)
self.set_run_after(slave)
def runnable_status(self):
for t in self.run_after:
if not t.hasrun:
return Task.ASK_LATER
for t in self.slaves:
#if t.executed:
if t.hasrun != Task.SKIPPED:
return Task.RUN_ME
return Task.SKIP_ME
def get_cwd(self):
return self.slaves[0].outputs[0].parent
def batch_incpaths(self):
st = self.env.CPPPATH_ST
return [st % node.abspath() for node in self.generator.includes_nodes]
def run(self):
self.outputs = []
srclst = []
slaves = []
for t in self.slaves:
if t.hasrun != Task.SKIPPED:
slaves.append(t)
srclst.append(t.inputs[0].abspath())
self.env.SRCLST = srclst
if self.slaves[0].__class__.__name__ == 'c':
ret = c_fun(self)
else:
ret = cxx_fun(self)
if ret:
return ret
for t in slaves:
t.old_post_run()
def hook(cls_type):
def n_hook(self, node):
ext = '.obj' if self.env.CC_NAME == 'msvc' else '.o'
name = node.name
k = name.rfind('.')
if k >= 0:
basename = name[:k] + ext
else:
basename = name + ext
outdir = node.parent.get_bld().make_node('%d' % self.idx)
outdir.mkdir()
out = outdir.find_or_declare(basename)
task = self.create_task(cls_type, node, out)
try:
self.compiled_tasks.append(task)
except AttributeError:
self.compiled_tasks = [task]
if not getattr(self, 'masters', None):
self.masters = {}
self.allmasters = []
def fix_path(tsk):
if self.env.CC_NAME == 'msvc':
tsk.env.append_unique('CXX_TGT_F_BATCHED', '/Fo%s\\' % outdir.abspath())
if not node.parent in self.masters:
m = self.masters[node.parent] = self.master = self.create_task('batch')
fix_path(m)
self.allmasters.append(m)
else:
m = self.masters[node.parent]
if len(m.slaves) > MAX_BATCH:
m = self.masters[node.parent] = self.master = self.create_task('batch')
fix_path(m)
self.allmasters.append(m)
m.add_slave(task)
return task
return n_hook
extension('.c')(hook('c'))
extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx'))
@feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib')
@after_method('apply_link')
def link_after_masters(self):
if getattr(self, 'allmasters', None):
for m in self.allmasters:
self.link_task.set_run_after(m)
# Modify the c and cxx task classes - in theory it would be best to
# create subclasses and to re-map the c/c++ extensions
for x in ('c', 'cxx'):
t = Task.classes[x]
def run(self):
pass
def post_run(self):
pass
setattr(t, 'oldrun', getattr(t, 'run', None))
setattr(t, 'run', run)
setattr(t, 'old_post_run', t.post_run)
setattr(t, 'post_run', post_run)
|
