1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
|
# @file Edk2ToolsBuild.py
# Invocable class that builds the basetool c files.
#
# Supports VS2019, VS2022, and GCC5
##
# Copyright (c) Microsoft Corporation
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import os
import sys
import logging
import argparse
import multiprocessing
import shutil
from edk2toolext import edk2_logging
from edk2toolext.environment import self_describing_environment, shell_environment
from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
from edk2toollib.utility_functions import RunCmd, GetHostInfo
from edk2toollib.windows.locate_tools import QueryVcVariables
class Edk2ToolsBuild(BaseAbstractInvocable):
def ParseCommandLineOptions(self):
''' parse arguments '''
ParserObj = argparse.ArgumentParser()
ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",
help="Set the toolchain used to compile the build tools")
ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
"behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
args = ParserObj.parse_args()
self.tool_chain_tag = args.tct
self.target_arch = args.arch
def GetWorkspaceRoot(self):
''' Return the workspace root for initializing the SDE '''
# this is the bastools dir...not the traditional EDK2 workspace root
return os.path.dirname(os.path.abspath(__file__))
def GetActiveScopes(self):
''' return tuple containing scopes that should be active for this process '''
# Adding scope for cross compilers when building for ARM/AARCH64
scopes = ('global',)
if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"):
if self.target_arch is None:
return scopes
if "AARCH64" in self.target_arch:
scopes += ("gcc_aarch64_linux",)
if "ARM" in self.target_arch:
scopes += ("gcc_arm_linux",)
return scopes
def GetLoggingLevel(self, loggerType):
''' Get the logging level for a given type (return Logging.Level)
base == lowest logging level supported
con == Screen logging
txt == plain text file logging
md == markdown file logging
'''
if(loggerType == "con"):
return logging.ERROR
else:
return logging.DEBUG
def GetLoggingFolderRelativeToRoot(self):
''' Return a path to folder for log files '''
return "BaseToolsBuild"
def GetVerifyCheckRequired(self):
''' Will call self_describing_environment.VerifyEnvironment if this returns True '''
return True
def GetLoggingFileName(self, loggerType):
''' Get the logging file name for the type.
Return None if the logger shouldn't be created
base == lowest logging level supported
con == Screen logging
txt == plain text file logging
md == markdown file logging
'''
return "BASETOOLS_BUILD"
def WritePathEnvFile(self, OutputDir):
''' Write a PyTool path env file for future PyTool based edk2 builds'''
content = '''##
# Set shell variable EDK_TOOLS_BIN to this folder
#
# Autogenerated by Edk2ToolsBuild.py
#
# Copyright (c), Microsoft Corporation
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"id": "You-Built-BaseTools",
"scope": "edk2-build",
"flags": ["set_shell_var", "set_path"],
"var_name": "EDK_TOOLS_BIN"
}
'''
with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f:
f.write(content)
def Go(self):
logging.info("Running Python version: " + str(sys.version_info))
(build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
self.GetWorkspaceRoot(), self.GetActiveScopes())
# # Bind our current execution environment into the shell vars.
ph = os.path.dirname(sys.executable)
if " " in ph:
ph = '"' + ph + '"'
shell_env.set_shell_var("PYTHON_HOME", ph)
# PYTHON_COMMAND is required to be set for using edk2 python builds.
pc = sys.executable
if " " in pc:
pc = '"' + pc + '"'
shell_env.set_shell_var("PYTHON_COMMAND", pc)
if self.tool_chain_tag.lower().startswith("vs"):
if self.target_arch is None:
# Put a default as IA32
self.target_arch = "IA32"
if self.target_arch == "IA32":
VcToolChainArch = "x86"
TargetInfoArch = "x86"
OutputDir = "Win32"
elif self.target_arch == "ARM":
VcToolChainArch = "x86_arm"
TargetInfoArch = "ARM"
OutputDir = "Win32"
elif self.target_arch == "X64":
VcToolChainArch = "amd64"
TargetInfoArch = "x86"
OutputDir = "Win64"
elif self.target_arch == "AARCH64":
VcToolChainArch = "amd64_arm64"
TargetInfoArch = "ARM"
OutputDir = "Win64"
else:
raise NotImplementedError()
self.OutputDir = os.path.join(
shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)
# compiled tools need to be added to path because antlr is referenced
HostInfo = GetHostInfo()
if TargetInfoArch == HostInfo.arch:
# not cross compiling
shell_env.insert_path(self.OutputDir)
else:
# cross compiling:
# as the VfrCompile tool is needed in the build process, we need
# to build one for the host system, then add the path to the
# tools to the PATH environment variable
shell_environment.CheckpointBuildVars()
if HostInfo.arch == "x86" and HostInfo.bit == "64":
host_arch = "X64"
host_toolchain_arch = "amd64"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
elif HostInfo.arch == "x86" and HostInfo.bit == "32":
host_arch = "IA32"
host_toolchain_arch = "x86"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
elif HostInfo.arch == "ARM" and HostInfo.bit == "64":
host_arch = "AARCH64"
host_toolchain_arch = "amd64_arm64"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
elif HostInfo.arch == "ARM" and HostInfo.bit == "32":
host_arch = "ARM"
host_toolchain_arch = "x86_arm"
TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
else:
raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))
interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
interesting_keys.extend(
["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
interesting_keys.extend(
["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
vc_vars = QueryVcVariables(
interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())
for key in vc_vars.keys():
logging.debug(f"Var - {key} = {vc_vars[key]}")
if key.lower() == 'path':
shell_env.set_path(vc_vars[key])
else:
shell_env.set_shell_var(key, vc_vars[key])
# Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
# this script is BUILDING the base tools.
shell_env.set_shell_var('HOST_ARCH', host_arch)
shell_env.insert_path(TempOutputDir)
# All set, build the tools for the host system.
ret = RunCmd('nmake.exe', None,
workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
if ret != 0:
raise Exception("Failed to build base tools for host system.")
# Copy the output to a temp directory
TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")
if not os.path.exists(TempFolder):
os.makedirs(TempFolder)
for file in os.listdir(TempOutputDir):
shutil.copy(os.path.join(TempOutputDir, file), TempFolder)
# Clean up the build output
ret = RunCmd('nmake.exe', 'cleanall',
workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
# Remove the entire TempOutputDir
shutil.rmtree(TempOutputDir)
shell_environment.RevertBuildVars()
shell_env.insert_path(TempFolder)
# # Update environment with required VC vars.
interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
interesting_keys.extend(
["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
interesting_keys.extend(
["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
vc_vars = QueryVcVariables(
interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())
for key in vc_vars.keys():
logging.debug(f"Var - {key} = {vc_vars[key]}")
if key.lower() == 'path':
shell_env.set_path(vc_vars[key])
else:
shell_env.set_shell_var(key, vc_vars[key])
# Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
# this script is BUILDING the base tools.
shell_env.set_shell_var('HOST_ARCH', self.target_arch)
# Actually build the tools.
output_stream = edk2_logging.create_output_stream()
ret = RunCmd('nmake.exe', None,
workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
edk2_logging.remove_output_stream(output_stream)
problems = edk2_logging.scan_compiler_output(output_stream)
for level, problem in problems:
logging.log(level, problem)
if ret != 0:
raise Exception("Failed to build.")
self.WritePathEnvFile(self.OutputDir)
return ret
elif self.tool_chain_tag.lower().startswith("gcc"):
# Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
# this script is BUILDING the base tools.
HostInfo = GetHostInfo()
prefix = None
TargetInfoArch = None
if self.target_arch is not None:
shell_env.set_shell_var('HOST_ARCH', self.target_arch)
if "AARCH64" in self.target_arch:
prefix = shell_env.get_shell_var("GCC5_AARCH64_PREFIX")
if prefix == None:
# now check for install dir. If set then set the Prefix
install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL")
# make GCC5_AARCH64_PREFIX to align with tools_def.txt
prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")
shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
TargetInfoArch = "ARM"
elif "ARM" in self.target_arch:
prefix = shell_env.get_shell_var("GCC5_ARM_PREFIX")
if prefix == None:
# now check for install dir. If set then set the Prefix
install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")
# make GCC5_ARM_PREFIX to align with tools_def.txt
prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")
shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
TargetInfoArch = "ARM"
else:
TargetInfoArch = "x86"
else:
self.target_arch = HostInfo.arch
TargetInfoArch = HostInfo.arch
# Otherwise, the built binary arch will be consistent with the host system
# Added logic to support cross compilation scenarios
if TargetInfoArch != HostInfo.arch:
# this is defaulting to the version that comes with Ubuntu 20.04
ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34")
work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot())
pack_name = f"util-linux-{ver}"
unzip_dir = os.path.join(work_dir, pack_name)
if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")):
os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz"))
if os.path.isdir(unzip_dir):
shutil.rmtree(unzip_dir)
# cross compiling, need to rebuild libuuid for the target
ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir)
if ret != 0:
raise Exception(f"Failed to download libuuid version {ver} - {ret}")
ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir)
if ret != 0:
raise Exception(f"Failed to untar the downloaded file {ret}")
# configure the source to use the cross compiler
pack_name = f"util-linux-{ver}"
if "AARCH64" in self.target_arch:
ret = RunCmd("sh", f"./configure --host=aarch64-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
elif "ARM" in self.target_arch:
ret = RunCmd("sh", f"./configure --host=arm-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
if ret != 0:
raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}")
ret = RunCmd("make", "", workingdir=unzip_dir)
if ret != 0:
raise Exception(f"Failed to build the libuuid with our gcc {ret}")
shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir)
shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src"))
ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
if ret != 0:
raise Exception("Failed to build.")
cpu_count = self.GetCpuThreads()
output_stream = edk2_logging.create_output_stream()
ret = RunCmd("make", f"-C . -j {cpu_count}", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
edk2_logging.remove_output_stream(output_stream)
problems = edk2_logging.scan_compiler_output(output_stream)
for level, problem in problems:
logging.log(level, problem)
if ret != 0:
raise Exception("Failed to build.")
self.OutputDir = os.path.join(
shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin")
self.WritePathEnvFile(self.OutputDir)
return ret
logging.critical("Tool Chain not supported")
return -1
def GetCpuThreads(self) -> int:
''' Function to return number of cpus. If error return 1'''
cpus = 1
try:
cpus = multiprocessing.cpu_count()
except:
# from the internet there are cases where cpu_count is not implemented.
# will handle error by just doing single proc build
pass
return cpus
def main():
Edk2ToolsBuild().Invoke()
if __name__ == "__main__":
main()
|
