1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
# SPDX-License-Identifier: Apache-2.0
# Copyright 2018 The Meson development team
from __future__ import annotations
import asyncio.subprocess
import fnmatch
import itertools
import json
import signal
import sys
from pathlib import Path
from .. import mlog
from ..compilers import lang_suffixes
from ..mesonlib import quiet_git, join_args, determine_worker_count
from ..mtest import complete_all
import typing as T
Info = T.TypeVar("Info")
async def run_with_buffered_output(cmdlist: T.List[str], env: T.Optional[T.Dict[str, str]] = None) -> int:
"""Run the command in cmdlist, buffering the output so that it is
not mixed for multiple child processes. Kill the child on
cancellation."""
quoted_cmdline = join_args(cmdlist)
p: T.Optional[asyncio.subprocess.Process] = None
try:
p = await asyncio.create_subprocess_exec(*cmdlist, env=env,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT)
stdo, _ = await p.communicate()
except FileNotFoundError as e:
print(mlog.blue('>>>'), quoted_cmdline, file=sys.stderr)
print(mlog.red('not found:'), e.filename, file=sys.stderr)
return 1
except asyncio.CancelledError:
if p:
p.kill()
await p.wait()
return p.returncode or 1
else:
return 0
if stdo:
print(mlog.blue('>>>'), quoted_cmdline, flush=True)
sys.stdout.buffer.write(stdo)
return p.returncode
async def _run_workers(infos: T.Iterable[Info],
fn: T.Callable[[Info], T.Iterable[T.Coroutine[None, None, int]]]) -> int:
futures: T.List[asyncio.Future[int]] = []
semaphore = asyncio.Semaphore(determine_worker_count())
async def run_one(worker_coro: T.Coroutine[None, None, int]) -> int:
try:
async with semaphore:
return await worker_coro
except asyncio.CancelledError as e:
worker_coro.throw(e)
return await worker_coro
def sigterm_handler() -> None:
for f in futures:
f.cancel()
if sys.platform != 'win32':
loop = asyncio.get_running_loop()
loop.add_signal_handler(signal.SIGINT, sigterm_handler)
loop.add_signal_handler(signal.SIGTERM, sigterm_handler)
for i in infos:
futures.extend((asyncio.ensure_future(run_one(x)) for x in fn(i)))
if not futures:
return 0
try:
await complete_all(futures)
except BaseException:
for f in futures:
f.cancel()
raise
return max(f.result() for f in futures if f.done() and not f.cancelled())
def parse_pattern_file(fname: Path) -> T.List[str]:
patterns = []
try:
with fname.open(encoding='utf-8') as f:
for line in f:
pattern = line.strip()
if pattern and not pattern.startswith('#'):
patterns.append(pattern)
except FileNotFoundError:
pass
return patterns
def all_clike_files(name: str, srcdir: Path, builddir: Path) -> T.Iterable[Path]:
patterns = parse_pattern_file(srcdir / f'.{name}-include')
globs: T.Sequence[T.Union[T.List[Path], T.Iterator[Path], T.Generator[Path, None, None]]]
if patterns:
globs = [srcdir.glob(p) for p in patterns]
else:
r, o = quiet_git(['ls-files'], srcdir)
if r:
globs = [[Path(srcdir, f) for f in o.splitlines()]]
else:
globs = [srcdir.glob('**/*')]
patterns = parse_pattern_file(srcdir / f'.{name}-ignore')
ignore = [str(builddir / '*')]
ignore.extend([str(srcdir / p) for p in patterns])
suffixes = set(lang_suffixes['c']).union(set(lang_suffixes['cpp']))
suffixes.add('h')
suffixes = {f'.{s}' for s in suffixes}
for f in itertools.chain.from_iterable(globs):
strf = str(f)
if f.is_dir() or f.suffix not in suffixes or \
any(fnmatch.fnmatch(strf, i) for i in ignore):
continue
yield f
def run_clang_tool(name: str, srcdir: Path, builddir: Path, fn: T.Callable[..., T.Coroutine[None, None, int]], *args: T.Any) -> int:
if sys.platform == 'win32' and sys.version_info < (3, 8):
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
def wrapper(path: Path) -> T.Iterable[T.Coroutine[None, None, int]]:
yield fn(path, *args)
return asyncio.run(_run_workers(all_clike_files(name, srcdir, builddir), wrapper))
def run_clang_tool_on_sources(name: str, srcdir: Path, builddir: Path, fn: T.Callable[..., T.Coroutine[None, None, int]], *args: T.Any) -> int:
if sys.platform == 'win32' and sys.version_info < (3, 8):
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
source_files = set()
with open('meson-info/intro-targets.json', encoding='utf-8') as fp:
targets = json.load(fp)
for target in targets:
for target_source in target.get('target_sources') or []:
for source in target_source.get('sources') or []:
source_files.add(Path(source))
clike_files = set(all_clike_files(name, srcdir, builddir))
source_files = source_files.intersection(clike_files)
def wrapper(path: Path) -> T.Iterable[T.Coroutine[None, None, int]]:
yield fn(path, *args)
return asyncio.run(_run_workers(source_files, wrapper))
def run_tool_on_targets(fn: T.Callable[[T.Dict[str, T.Any]],
T.Iterable[T.Coroutine[None, None, int]]]) -> int:
if sys.platform == 'win32' and sys.version_info < (3, 8):
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
with open('meson-info/intro-targets.json', encoding='utf-8') as fp:
targets = json.load(fp)
return asyncio.run(_run_workers(targets, fn))
|
