1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
|
#!/usr/bin/env python3
#
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Contains general-purpose methods that can be used to execute shell,
GN and Ninja commands.
"""
import shlex
import subprocess
import os
import re
import pathlib
import difflib
from typing import Set, List
REPOSITORY_ROOT = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))
_MB_PATH = os.path.join(REPOSITORY_ROOT, 'tools/mb/mb.py')
GN_PATH = os.path.join(REPOSITORY_ROOT, 'buildtools/linux64/gn')
NINJA_PATH = os.path.join(REPOSITORY_ROOT, 'third_party/ninja/ninja')
MIN_SDK_VERSION_FOR_AOSP = 30
ARCHS = ['x86', 'x64', 'arm', 'arm64', 'riscv64']
AOSP_EXTRA_ARGS = ('is_cronet_for_aosp_build=true', 'use_nss_certs=false',
'use_allocator_shim=false',
f'default_min_sdk_version={MIN_SDK_VERSION_FOR_AOSP}')
_GN_ARG_MATCHER = re.compile("^.*=.*$")
def run(command, **kwargs):
"""See the official documentation for subprocess.check_call.
Args:
command (list[str]): command to be executed
"""
print('Executing: ' + ' '.join(shlex.quote(arg) for arg in command))
subprocess.check_call(command, **kwargs)
def run_and_get_stdout(command, **kwargs):
"""See the official documentation for subprocess.run.
Args:
command (list[str]): command to be executed
Returns:
str: stdout for the executed command
"""
print('Executing: ' + ' '.join(shlex.quote(arg) for arg in command))
return subprocess.run(command, capture_output=True,
check=True, **kwargs).stdout.decode('utf-8').strip()
def gn(out_dir, gn_args, gn_extra=None, **kwargs):
""" Executes `gn gen`.
Runs `gn gen |out_dir| |gn_args + gn_extra|` which will generate
a GN configuration that lives under |out_dir|. This is done
locally on the same chromium checkout.
Args:
out_dir (str): Path to delegate to `gn gen`.
gn_args (str): Args as a string delimited by space.
gn_extra (str): extra args as a string delimited by space.
"""
cmd = [GN_PATH, 'gen', out_dir, '--args=%s' % gn_args]
if gn_extra:
cmd += gn_extra
run(cmd, **kwargs)
def compare_text_and_generate_diff(generated_text, golden_text,
golden_file_path):
"""
Compares the generated text with the golden text.
returns a diff that can be applied with `patch` if exists.
"""
golden_lines = [line.rstrip() for line in golden_text.splitlines()]
generated_lines = [line.rstrip() for line in generated_text.splitlines()]
if golden_lines == generated_lines:
return None
expected_path = os.path.relpath(golden_file_path, REPOSITORY_ROOT)
diff = difflib.unified_diff(
golden_lines,
generated_lines,
fromfile=os.path.join('before', expected_path),
tofile=os.path.join('after', expected_path),
n=0,
lineterm='',
)
return '\n'.join(diff)
def read_file(path):
"""Reads a file as a string"""
return pathlib.Path(path).read_text()
def write_file(path, contents):
"""Writes contents to a file"""
return pathlib.Path(path).write_text(contents)
def build(out_dir, build_target, extra_options=None):
"""Runs `ninja build`.
Runs `ninja -C |out_dir| |build_target| |extra_options|` which will build
the target |build_target| for the GN configuration living under |out_dir|.
This is done locally on the same chromium checkout.
"""
cmd = [NINJA_PATH, '-C', out_dir, build_target]
if extra_options:
cmd += extra_options
run(cmd)
def build_all(out_dir, build_targets, extra_options=None):
"""Runs `ninja build`.
Runs `ninja -C |out_dir| |build_targets| |extra_options|` which will build
the targets |build_targets| for the GN configuration living under |out_dir|.
This is done locally on the same chromium checkout.
"""
cmd = [NINJA_PATH, '-C', out_dir]
cmd.extend(build_targets)
if extra_options:
cmd += extra_options
run(cmd)
def get_transitive_deps_build_files(repo_path: str, out_dir: str,
gn_targets: List[str]) -> Set[str]:
"""Executes gn desc |out_dir| |gn_target| deps --all --as=buildfile for each gn target"""
all_deps = set()
for gn_target in gn_targets:
all_deps.update(
subprocess.check_output([
GN_PATH, "desc", out_dir, gn_target, "deps", "--all",
"--as=buildfile"
]).decode("utf-8").split("\n"))
# gn desc deps does not return the build file that includes the target
# which we want to find its transitive dependencies, in order to
# account for this corner case, the BUILD file for the current target
# is added manually.
all_deps.add(
f"{os.path.join(repo_path, gn_target[2:gn_target.find(':')])}/BUILD.gn")
# It seems that we always get an empty string as part of the output. This
# could happen if we get an empty line in the output which can happen so
# let's remove that so downstream consumers don't have to check for it.
all_deps.remove('')
return all_deps
def get_gn_args_for_aosp(arch: str) -> List[str]:
default_args = filter_gn_args(get_android_gn_args(True, arch),
["use_remoteexec", "default_min_sdk_version"])
default_args.extend(AOSP_EXTRA_ARGS)
return default_args
def android_gn_gen(is_release, target_cpu, out_dir):
"""Runs `gn gen` using Cronet's android gn_args.
Creates a local GN configuration under |out_dir| with the provided argument
as input to `get_android_gn_args`, see the documentation of
`get_android_gn_args` for more information.
"""
return gn(out_dir, ' '.join(get_android_gn_args(is_release, target_cpu)))
def get_android_gn_args(is_release, target_cpu):
"""Fetches the gn args for a specific builder.
Returns a list of gn args used by the builders whose target cpu
is |target_cpu| and (dev or rel) depending on is_release.
See https://ci.chromium.org/p/chromium/g/chromium.android/console for
a list of the builders
Example:
get_android_gn_args(true, 'x86') -> GN Args for `android-cronet-x86-rel`
get_android_gn_args(false, 'x86') -> GN Args for `android-cronet-x86-dev`
"""
group_name = 'chromium.android'
builder_name = _map_config_to_android_builder(is_release, target_cpu)
# Ideally we would call `mb_py gen` directly, but we need to filter out the
# use_remoteexec arg, as that cannot be used in a local environment.
gn_args = subprocess.check_output(
['python3', _MB_PATH, 'lookup', '-m', group_name, '-b',
builder_name]).decode('utf-8').strip()
return filter_gn_args(gn_args.split("\n"), [])
def get_path_from_gn_label(gn_label: str) -> str:
"""Returns the path part from a GN Label
GN label consist of two parts, path and target_name, this will
remove the target name and return the path or throw an error
if it can't remove the target_name or if it doesn't exist.
"""
if ":" not in gn_label:
raise ValueError(f"Provided gn label {gn_label} is not a proper label")
return gn_label[:gn_label.find(":")]
def _map_config_to_android_builder(is_release, target_cpu):
target_cpu_to_base_builder = {
'x86': 'android-cronet-x86',
'x64': 'android-cronet-x64',
'arm': 'android-cronet-arm',
'arm64': 'android-cronet-arm64',
'riscv64': 'android-cronet-riscv64',
}
if target_cpu not in target_cpu_to_base_builder:
raise ValueError('Unsupported target CPU')
builder_name = target_cpu_to_base_builder[target_cpu]
if is_release:
builder_name += '-rel'
else:
builder_name += '-dbg'
return builder_name
def _should_remove_arg(arg, keys):
"""An arg is removed if its key appear in the list of |keys|"""
return arg.split("=")[0].strip() in keys
def filter_gn_args(gn_args, keys_to_remove):
"""Returns a list of filtered GN args.
(1) GN arg's returned must match the regex |_GN_ARG_MATCHER|.
(2) GN arg's key must not be in |keys_to_remove|.
Args:
gn_args: list of GN args.
keys_to_remove: List of string that will be removed from gn_args.
"""
filtered_args = []
for arg in gn_args:
if _GN_ARG_MATCHER.match(arg) and not _should_remove_arg(
arg, keys_to_remove):
filtered_args.append(arg)
return filtered_args
|