File: utils.py

package info (click to toggle)
chromium 139.0.7258.127-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 6,122,156 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (253 lines) | stat: -rwxr-xr-x 8,312 bytes parent folder | download | duplicates (5)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
#!/usr/bin/env python3
#
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Contains general-purpose methods that can be used to execute shell,
GN and Ninja commands.
"""

import shlex
import subprocess
import os
import re
import pathlib
import difflib
from typing import Set, List

REPOSITORY_ROOT = os.path.abspath(
    os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir))

_MB_PATH = os.path.join(REPOSITORY_ROOT, 'tools/mb/mb.py')
GN_PATH = os.path.join(REPOSITORY_ROOT, 'buildtools/linux64/gn')
NINJA_PATH = os.path.join(REPOSITORY_ROOT, 'third_party/ninja/ninja')
MIN_SDK_VERSION_FOR_AOSP = 30
ARCHS = ['x86', 'x64', 'arm', 'arm64', 'riscv64']
AOSP_EXTRA_ARGS = ('is_cronet_for_aosp_build=true', 'use_nss_certs=false',
                   'use_allocator_shim=false',
                   f'default_min_sdk_version={MIN_SDK_VERSION_FOR_AOSP}')
_GN_ARG_MATCHER = re.compile("^.*=.*$")


def run(command, **kwargs):
  """See the official documentation for subprocess.check_call.

  Args:
    command (list[str]): command to be executed
  """
  print('Executing: ' + ' '.join(shlex.quote(arg) for arg in command))
  subprocess.check_call(command, **kwargs)


def run_and_get_stdout(command, **kwargs):
  """See the official documentation for subprocess.run.

  Args:
    command (list[str]): command to be executed

  Returns:
    str: stdout for the executed command
  """
  print('Executing: ' + ' '.join(shlex.quote(arg) for arg in command))
  return subprocess.run(command, capture_output=True,
                        check=True, **kwargs).stdout.decode('utf-8').strip()


def gn(out_dir, gn_args, gn_extra=None, **kwargs):
  """ Executes `gn gen`.

  Runs `gn gen |out_dir| |gn_args + gn_extra|` which will generate
  a GN configuration that lives under |out_dir|. This is done
  locally on the same chromium checkout.

  Args:
    out_dir (str): Path to delegate to `gn gen`.
    gn_args (str): Args as a string delimited by space.
    gn_extra (str): extra args as a string delimited by space.
  """
  cmd = [GN_PATH, 'gen', out_dir, '--args=%s' % gn_args]
  if gn_extra:
    cmd += gn_extra
  run(cmd, **kwargs)


def compare_text_and_generate_diff(generated_text, golden_text,
                                   golden_file_path):
  """
  Compares the generated text with the golden text.

  returns a diff that can be applied with `patch` if exists.
  """
  golden_lines = [line.rstrip() for line in golden_text.splitlines()]
  generated_lines = [line.rstrip() for line in generated_text.splitlines()]
  if golden_lines == generated_lines:
    return None

  expected_path = os.path.relpath(golden_file_path, REPOSITORY_ROOT)

  diff = difflib.unified_diff(
      golden_lines,
      generated_lines,
      fromfile=os.path.join('before', expected_path),
      tofile=os.path.join('after', expected_path),
      n=0,
      lineterm='',
  )

  return '\n'.join(diff)


def read_file(path):
  """Reads a file as a string"""
  return pathlib.Path(path).read_text()


def write_file(path, contents):
  """Writes contents to a file"""
  return pathlib.Path(path).write_text(contents)


def build(out_dir, build_target, extra_options=None):
  """Runs `ninja build`.

  Runs `ninja -C |out_dir| |build_target| |extra_options|` which will build
  the target |build_target| for the GN configuration living under |out_dir|.
  This is done locally on the same chromium checkout.
  """
  cmd = [NINJA_PATH, '-C', out_dir, build_target]
  if extra_options:
    cmd += extra_options
  run(cmd)


def build_all(out_dir, build_targets, extra_options=None):
  """Runs `ninja build`.

  Runs `ninja -C |out_dir| |build_targets| |extra_options|` which will build
  the targets |build_targets| for the GN configuration living under |out_dir|.
  This is done locally on the same chromium checkout.
  """
  cmd = [NINJA_PATH, '-C', out_dir]
  cmd.extend(build_targets)
  if extra_options:
    cmd += extra_options
  run(cmd)


def get_transitive_deps_build_files(repo_path: str, out_dir: str,
                                    gn_targets: List[str]) -> Set[str]:
  """Executes gn desc |out_dir| |gn_target| deps --all --as=buildfile for each gn target"""
  all_deps = set()
  for gn_target in gn_targets:
    all_deps.update(
        subprocess.check_output([
            GN_PATH, "desc", out_dir, gn_target, "deps", "--all",
            "--as=buildfile"
        ]).decode("utf-8").split("\n"))
    # gn desc deps does not return the build file that includes the target
    # which we want to find its transitive dependencies, in order to
    # account for this corner case, the BUILD file for the current target
    # is added manually.
    all_deps.add(
        f"{os.path.join(repo_path, gn_target[2:gn_target.find(':')])}/BUILD.gn")
  # It seems that we always get an empty string as part of the output. This
  # could happen if we get an empty line in the output which can happen so
  # let's remove that so downstream consumers don't have to check for it.
  all_deps.remove('')
  return all_deps


def get_gn_args_for_aosp(arch: str) -> List[str]:
  default_args = filter_gn_args(get_android_gn_args(True, arch),
                                ["use_remoteexec", "default_min_sdk_version"])
  default_args.extend(AOSP_EXTRA_ARGS)
  return default_args


def android_gn_gen(is_release, target_cpu, out_dir):
  """Runs `gn gen` using Cronet's android gn_args.

  Creates a local GN configuration under |out_dir| with the provided argument
  as input to `get_android_gn_args`, see the documentation of
  `get_android_gn_args` for more information.
  """
  return gn(out_dir, ' '.join(get_android_gn_args(is_release, target_cpu)))


def get_android_gn_args(is_release, target_cpu):
  """Fetches the gn args for a specific builder.

  Returns a list of gn args used by the builders whose target cpu
  is |target_cpu| and (dev or rel) depending on is_release.

  See https://ci.chromium.org/p/chromium/g/chromium.android/console for
  a list of the builders

  Example:

  get_android_gn_args(true, 'x86') -> GN Args for `android-cronet-x86-rel`
  get_android_gn_args(false, 'x86') -> GN Args for `android-cronet-x86-dev`
  """
  group_name = 'chromium.android'
  builder_name = _map_config_to_android_builder(is_release, target_cpu)
  # Ideally we would call `mb_py gen` directly, but we need to filter out the
  # use_remoteexec arg, as that cannot be used in a local environment.
  gn_args = subprocess.check_output(
      ['python3', _MB_PATH, 'lookup', '-m', group_name, '-b',
       builder_name]).decode('utf-8').strip()
  return filter_gn_args(gn_args.split("\n"), [])


def get_path_from_gn_label(gn_label: str) -> str:
  """Returns the path part from a GN Label

  GN label consist of two parts, path and target_name, this will
  remove the target name and return the path or throw an error
  if it can't remove the target_name or if it doesn't exist.
  """
  if ":" not in gn_label:
    raise ValueError(f"Provided gn label {gn_label} is not a proper label")
  return gn_label[:gn_label.find(":")]


def _map_config_to_android_builder(is_release, target_cpu):
  target_cpu_to_base_builder = {
      'x86': 'android-cronet-x86',
      'x64': 'android-cronet-x64',
      'arm': 'android-cronet-arm',
      'arm64': 'android-cronet-arm64',
      'riscv64': 'android-cronet-riscv64',
  }
  if target_cpu not in target_cpu_to_base_builder:
    raise ValueError('Unsupported target CPU')

  builder_name = target_cpu_to_base_builder[target_cpu]
  if is_release:
    builder_name += '-rel'
  else:
    builder_name += '-dbg'
  return builder_name


def _should_remove_arg(arg, keys):
  """An arg is removed if its key appear in the list of |keys|"""
  return arg.split("=")[0].strip() in keys


def filter_gn_args(gn_args, keys_to_remove):
  """Returns a list of filtered GN args.

  (1) GN arg's returned must match the regex |_GN_ARG_MATCHER|.
  (2) GN arg's key must not be in |keys_to_remove|.

  Args:
    gn_args: list of GN args.
    keys_to_remove: List of string that will be removed from gn_args.
  """
  filtered_args = []
  for arg in gn_args:
    if _GN_ARG_MATCHER.match(arg) and not _should_remove_arg(
        arg, keys_to_remove):
      filtered_args.append(arg)
  return filtered_args