1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
|
# Copyright 2024 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import logging
import os
import pathlib
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.request
import scripthash
_THIS_DIR = pathlib.Path(__file__).resolve().parent
_SRC_ROOT = _THIS_DIR.parents[1]
_CHECKOUT_SRC_ROOT_SUBDIR = '.3pp/chromium'
def parse_args():
parser = argparse.ArgumentParser()
# TODO(agrieve): Add required=True once 3pp builds with > python3.6.
subparsers = parser.add_subparsers()
subparser = subparsers.add_parser(
'latest', help='Prints the version as $LATEST.$RUNTIME_DEPS_HASH')
subparser.set_defaults(action='latest')
subparser = subparsers.add_parser(
'checkout', help='Copies files into the workdir used by docker')
subparser.add_argument('checkout_dir')
subparser.add_argument('--version', help='Output from "latest"')
subparser.set_defaults(action='checkout')
subparser = subparsers.add_parser(
'install',
help=('Run from workdir, inside docker container. '
'Builds & copies outputs into |output_prefix| directory'))
subparser.add_argument('output_prefix',
help='The path to install the compiled package to.')
subparser.add_argument('deps_prefix',
help='The path to a directory containing all deps.')
subparser.add_argument('--version', help='Output from "latest"')
subparser.add_argument('--checkout-dir', help='Directory to use as CWD')
subparser.set_defaults(action='install')
subparser = subparsers.add_parser(
'local-test', help='Run latest / checkout / install locally')
subparser.add_argument('--checkout-dir',
default='3pp_workdir',
help='Workdir to use')
subparser.add_argument('--output-prefix',
default='3pp_out',
help='Directory for final artifacts')
subparser.set_defaults(action='local-test')
args = parser.parse_args()
if not hasattr(args, 'action'):
parser.print_help()
sys.exit(1)
if hasattr(args, 'version'):
args.version = args.version or os.environ.get('_3PP_VERSION')
if not args.version:
parser.error('Must set --version or _3PP_VERSION')
if hasattr(args, 'output_prefix') and args.output_prefix:
args.output_prefix = os.path.abspath(args.output_prefix)
if hasattr(args, 'checkout_dir') and args.checkout_dir:
args.checkout_dir = os.path.abspath(args.checkout_dir)
if args.action == 'checkout':
# 3pp bot recipe does this, so needed only when running locally.
os.makedirs(args.checkout_dir, exist_ok=True)
if args.action == 'install':
if args.checkout_dir:
logging.info('Setting CWD=%s', args.checkout_dir)
os.chdir(args.checkout_dir)
if not os.path.exists(_CHECKOUT_SRC_ROOT_SUBDIR):
parser.error(f'Does not exist: {_CHECKOUT_SRC_ROOT_SUBDIR}.'
f' Use --checkout-dir?')
# 3pp bot recipe does this, so needed only when running locally.
os.makedirs(args.output_prefix, exist_ok=True)
return args
def path_within_checkout(subpath):
return os.path.abspath(os.path.join(_CHECKOUT_SRC_ROOT_SUBDIR, subpath))
def _all_files(path):
if os.path.isfile(path):
return [path]
assert os.path.isdir(path), 'Not a file or dir: ' + path
all_paths = pathlib.Path(path).glob('**/*')
return [str(f) for f in all_paths if f.is_file()]
def _resolve_runtime_deps(runtime_deps):
ret = []
for p in runtime_deps:
if p.startswith('//'):
ret.append(os.path.relpath(str(_SRC_ROOT / p[2:])))
elif os.path.isabs(p):
ret.append(os.path.relpath(p))
else:
ret.append(p)
return ret
def copy_runtime_deps(checkout_dir, runtime_deps):
# Make 3pp_common scripts available in the docker container install.py
# will run in.
dest_dir = os.path.join(checkout_dir, _CHECKOUT_SRC_ROOT_SUBDIR)
for src_path in _resolve_runtime_deps(runtime_deps):
relpath = os.path.relpath(src_path, _SRC_ROOT)
dest_path = os.path.join(dest_dir, relpath)
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
if os.path.isfile(src_path):
shutil.copy(src_path, dest_path)
else:
shutil.copytree(src_path,
dest_path,
ignore=shutil.ignore_patterns('.*', '__pycache__'))
logging.info('Runtime deps:')
sys.stderr.write('\n'.join(_all_files(checkout_dir)) + '\n')
def download_file(url, dest):
logging.info('Downloading %s', url)
with urllib.request.urlopen(url) as r:
with open(dest, 'wb') as f:
shutil.copyfileobj(r, f)
def extract_tar(path, dest):
logging.info('Extracting %s to %s', path, dest)
with tarfile.open(path) as f:
f.extractall(dest)
def run_cmd(cmd, check=True, *args, **kwargs):
logging.info('Running: %s', shlex.join(cmd))
return subprocess.run(cmd, check=check, *args, **kwargs)
def apply_patches(patches_dir, checkout_dir):
for path in sorted(pathlib.Path(patches_dir).glob('*.patch')):
cmd = ['git', 'apply', '-v', str(path)]
run_cmd(cmd, cwd=checkout_dir)
def main(*, do_latest, do_install, runtime_deps):
logging.basicConfig(
level=logging.DEBUG,
format='%(levelname).1s %(relativeCreated)6d %(message)s')
args = parse_args()
runtime_deps = [str(_THIS_DIR)] + runtime_deps
if args.action == 'local-test':
logging.warning('Will use work dir: %s', args.checkout_dir)
logging.warning('Will use output dir: %s', args.output_prefix)
if os.path.exists(args.checkout_dir) and os.listdir(args.checkout_dir):
logging.warning(
'*** Work dir not empty. This often causes failures. ***')
time.sleep(4)
# Approximates what 3pp recipe does for minimal configs.
# https://source.chromium.org/search?q=symbol:Chromium3ppApi.execute&ss=chromium
prog = os.path.abspath(sys.argv[0])
cmd = [prog, 'latest']
version = run_cmd(cmd, stdout=subprocess.PIPE, text=True).stdout
os.environ['_3PP_VERSION'] = version
checkout_dir = args.checkout_dir
run_cmd([prog, 'checkout', checkout_dir])
run_cmd([prog, 'install', args.output_prefix, 'UNUSED-DEPS-DIR'],
cwd=checkout_dir)
logging.warning('Local test complete.')
return
if args.action == 'latest':
version = do_latest()
assert version, 'do_latest() returned ' + repr(version)
extra_paths = []
for p in _resolve_runtime_deps(runtime_deps):
extra_paths += _all_files(p)
deps_hash = scripthash.compute(extra_paths=extra_paths)
print(f'{version}.{deps_hash}')
return
# Remove the hash at the end: 30.4.0-alpha05.HASH => 30.4.0-alpha05
args.version = args.version.rsplit('.', 1)[0]
if args.action == 'checkout':
copy_runtime_deps(args.checkout_dir, runtime_deps)
return
assert args.action == 'install'
do_install(args)
prefix_len = len(args.output_prefix) + 1
logging.info(
'Contents of %s: \n%s\n', args.output_prefix,
'\n'.join(p[prefix_len:] for p in _all_files(args.output_prefix)))
|
