1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
|
import logging
import sys
import textwrap
from c_common.scriptutil import (
VERBOSITY,
add_verbosity_cli,
add_traceback_cli,
add_commands_cli,
add_kind_filtering_cli,
add_files_cli,
add_progress_cli,
process_args_by_key,
configure_logger,
get_prog,
)
from c_parser.info import KIND
import c_parser.__main__ as c_parser
import c_analyzer.__main__ as c_analyzer
import c_analyzer as _c_analyzer
from c_analyzer.info import UNKNOWN
from . import _analyzer, _builtin_types, _capi, _files, _parser, REPO_ROOT
logger = logging.getLogger(__name__)
CHECK_EXPLANATION = textwrap.dedent('''
-------------------------
Non-constant global variables are generally not supported
in the CPython repo. We use a tool to analyze the C code
and report if any unsupported globals are found. The tool
may be run manually with:
./python Tools/c-analyzer/check-c-globals.py --format summary [FILE]
Occasionally the tool is unable to parse updated code.
If this happens then add the file to the "EXCLUDED" list
in Tools/c-analyzer/cpython/_parser.py and create a new
issue for fixing the tool (and CC ericsnowcurrently
on the issue).
If the tool reports an unsupported global variable and
it is actually const (and thus supported) then first try
fixing the declaration appropriately in the code. If that
doesn't work then add the variable to the "should be const"
section of Tools/c-analyzer/cpython/ignored.tsv.
If the tool otherwise reports an unsupported global variable
then first try to make it non-global, possibly adding to
PyInterpreterState (for core code) or module state (for
extension modules). In an emergency, you can add the
variable to Tools/c-analyzer/cpython/globals-to-fix.tsv
to get CI passing, but doing so should be avoided. If
this course it taken, be sure to create an issue for
eliminating the global (and CC ericsnowcurrently).
''')
def _resolve_filenames(filenames):
if filenames:
resolved = (_files.resolve_filename(f) for f in filenames)
else:
resolved = _files.iter_filenames()
return resolved
#######################################
# the formats
def fmt_summary(analysis):
# XXX Support sorting and grouping.
supported = []
unsupported = []
for item in analysis:
if item.supported:
supported.append(item)
else:
unsupported.append(item)
total = 0
def section(name, groupitems):
nonlocal total
items, render = c_analyzer.build_section(name, groupitems,
relroot=REPO_ROOT)
yield from render()
total += len(items)
yield ''
yield '===================='
yield 'supported'
yield '===================='
yield from section('types', supported)
yield from section('variables', supported)
yield ''
yield '===================='
yield 'unsupported'
yield '===================='
yield from section('types', unsupported)
yield from section('variables', unsupported)
yield ''
yield f'grand total: {total}'
#######################################
# the checks
CHECKS = dict(c_analyzer.CHECKS, **{
'globals': _analyzer.check_globals,
})
#######################################
# the commands
FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')
def _cli_parse(parser):
process_output = c_parser.add_output_cli(parser)
process_kind = add_kind_filtering_cli(parser)
process_preprocessor = c_parser.add_preprocessor_cli(
parser,
get_preprocessor=_parser.get_preprocessor,
)
process_files = add_files_cli(parser, **FILES_KWARGS)
return [
process_output,
process_kind,
process_preprocessor,
process_files,
]
def cmd_parse(filenames=None, **kwargs):
filenames = _resolve_filenames(filenames)
if 'get_file_preprocessor' not in kwargs:
kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
c_parser.cmd_parse(
filenames,
relroot=REPO_ROOT,
file_maxsizes=_parser.MAX_SIZES,
**kwargs
)
def _cli_check(parser, **kwargs):
return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)
def cmd_check(filenames=None, **kwargs):
filenames = _resolve_filenames(filenames)
kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
try:
c_analyzer.cmd_check(
filenames,
relroot=REPO_ROOT,
_analyze=_analyzer.analyze,
_CHECKS=CHECKS,
file_maxsizes=_parser.MAX_SIZES,
**kwargs
)
except SystemExit as exc:
num_failed = exc.args[0] if getattr(exc, 'args', None) else None
if isinstance(num_failed, int):
if num_failed > 0:
sys.stderr.flush()
print(CHECK_EXPLANATION, flush=True)
raise # re-raise
except Exception:
sys.stderr.flush()
print(CHECK_EXPLANATION, flush=True)
raise # re-raise
def cmd_analyze(filenames=None, **kwargs):
formats = dict(c_analyzer.FORMATS)
formats['summary'] = fmt_summary
filenames = _resolve_filenames(filenames)
kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
c_analyzer.cmd_analyze(
filenames,
relroot=REPO_ROOT,
_analyze=_analyzer.analyze,
formats=formats,
file_maxsizes=_parser.MAX_SIZES,
**kwargs
)
def _cli_data(parser):
filenames = False
known = True
return c_analyzer._cli_data(parser, filenames, known)
def cmd_data(datacmd, **kwargs):
formats = dict(c_analyzer.FORMATS)
formats['summary'] = fmt_summary
filenames = (file
for file in _resolve_filenames(None)
if file not in _parser.EXCLUDED)
kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
if datacmd == 'show':
types = _analyzer.read_known()
results = []
for decl, info in types.items():
if info is UNKNOWN:
if decl.kind in (KIND.STRUCT, KIND.UNION):
extra = {'unsupported': ['type unknown'] * len(decl.members)}
else:
extra = {'unsupported': ['type unknown']}
info = (info, extra)
results.append((decl, info))
if decl.shortkey == 'struct _object':
tempinfo = info
known = _analyzer.Analysis.from_results(results)
analyze = None
elif datacmd == 'dump':
known = _analyzer.KNOWN_FILE
def analyze(files, **kwargs):
decls = []
for decl in _analyzer.iter_decls(files, **kwargs):
if not KIND.is_type_decl(decl.kind):
continue
if not decl.filename.endswith('.h'):
if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
continue
decls.append(decl)
results = _c_analyzer.analyze_decls(
decls,
known={},
analyze_resolved=_analyzer.analyze_resolved,
)
return _analyzer.Analysis.from_results(results)
else: # check
known = _analyzer.read_known()
def analyze(files, **kwargs):
return _analyzer.iter_decls(files, **kwargs)
extracolumns = None
c_analyzer.cmd_data(
datacmd,
filenames,
known,
_analyze=analyze,
formats=formats,
extracolumns=extracolumns,
relroot=REPO_ROOT,
**kwargs
)
def _cli_capi(parser):
parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
parser.add_argument(f'--public', dest='levels',
action='append_const', const='public')
parser.add_argument(f'--no-public', dest='levels',
action='append_const', const='no-public')
for level in _capi.LEVELS:
parser.add_argument(f'--{level}', dest='levels',
action='append_const', const=level)
def process_levels(args, *, argv=None):
levels = []
for raw in args.levels or ():
for level in raw.replace(',', ' ').strip().split():
if level == 'public':
levels.append('stable')
levels.append('cpython')
elif level == 'no-public':
levels.append('private')
levels.append('internal')
elif level in _capi.LEVELS:
levels.append(level)
else:
parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
args.levels = set(levels)
parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
for kind in _capi.KINDS:
parser.add_argument(f'--{kind}', dest='kinds',
action='append_const', const=kind)
def process_kinds(args, *, argv=None):
kinds = []
for raw in args.kinds or ():
for kind in raw.replace(',', ' ').strip().split():
if kind in _capi.KINDS:
kinds.append(kind)
else:
parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
args.kinds = set(kinds)
parser.add_argument('--group-by', dest='groupby',
choices=['level', 'kind'])
parser.add_argument('--format', default='table')
parser.add_argument('--summary', dest='format',
action='store_const', const='summary')
def process_format(args, *, argv=None):
orig = args.format
args.format = _capi.resolve_format(args.format)
if isinstance(args.format, str):
if args.format not in _capi._FORMATS:
parser.error(f'unsupported format {orig!r}')
parser.add_argument('--show-empty', dest='showempty', action='store_true')
parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
parser.set_defaults(showempty=None)
# XXX Add --sort-by, --sort and --no-sort.
parser.add_argument('--ignore', dest='ignored', action='append')
def process_ignored(args, *, argv=None):
ignored = []
for raw in args.ignored or ():
ignored.extend(raw.replace(',', ' ').strip().split())
args.ignored = ignored or None
parser.add_argument('filenames', nargs='*', metavar='FILENAME')
process_progress = add_progress_cli(parser)
return [
process_levels,
process_kinds,
process_format,
process_ignored,
process_progress,
]
def cmd_capi(filenames=None, *,
levels=None,
kinds=None,
groupby='kind',
format='table',
showempty=None,
ignored=None,
track_progress=None,
verbosity=VERBOSITY,
**kwargs
):
render = _capi.get_renderer(format)
filenames = _files.iter_header_files(filenames, levels=levels)
#filenames = (file for file, _ in main_for_filenames(filenames))
if track_progress:
filenames = track_progress(filenames)
items = _capi.iter_capi(filenames)
if levels:
items = (item for item in items if item.level in levels)
if kinds:
items = (item for item in items if item.kind in kinds)
filter = _capi.resolve_filter(ignored)
if filter:
items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))
lines = render(
items,
groupby=groupby,
showempty=showempty,
verbose=verbosity > VERBOSITY,
)
print()
for line in lines:
print(line)
def _cli_builtin_types(parser):
parser.add_argument('--format', dest='fmt', default='table')
# parser.add_argument('--summary', dest='format',
# action='store_const', const='summary')
def process_format(args, *, argv=None):
orig = args.fmt
args.fmt = _builtin_types.resolve_format(args.fmt)
if isinstance(args.fmt, str):
if args.fmt not in _builtin_types._FORMATS:
parser.error(f'unsupported format {orig!r}')
parser.add_argument('--include-modules', dest='showmodules',
action='store_true')
def process_modules(args, *, argv=None):
pass
return [
process_format,
process_modules,
]
def cmd_builtin_types(fmt, *,
showmodules=False,
verbosity=VERBOSITY,
):
render = _builtin_types.get_renderer(fmt)
types = _builtin_types.iter_builtin_types()
match = _builtin_types.resolve_matcher(showmodules)
if match:
types = (t for t in types if match(t, log=lambda msg: logger.log(1, msg)))
lines = render(
types,
# verbose=verbosity > VERBOSITY,
)
print()
for line in lines:
print(line)
# We do not define any other cmd_*() handlers here,
# favoring those defined elsewhere.
COMMANDS = {
'check': (
'analyze and fail if the CPython source code has any problems',
[_cli_check],
cmd_check,
),
'analyze': (
'report on the state of the CPython source code',
[(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
cmd_analyze,
),
'parse': (
'parse the CPython source files',
[_cli_parse],
cmd_parse,
),
'data': (
'check/manage local data (e.g. known types, ignored vars, caches)',
[_cli_data],
cmd_data,
),
'capi': (
'inspect the C-API',
[_cli_capi],
cmd_capi,
),
'builtin-types': (
'show the builtin types',
[_cli_builtin_types],
cmd_builtin_types,
),
}
#######################################
# the script
def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
import argparse
parser = argparse.ArgumentParser(
prog=prog or get_prog(),
)
# if subset == 'check' or subset == ['check']:
# if checks is not None:
# commands = dict(COMMANDS)
# commands['check'] = list(commands['check'])
# cli = commands['check'][1][0]
# commands['check'][1][0] = (lambda p: cli(p, checks=checks))
processors = add_commands_cli(
parser,
commands=COMMANDS,
commonspecs=[
add_verbosity_cli,
add_traceback_cli,
],
subset=subset,
)
args = parser.parse_args(argv)
ns = vars(args)
cmd = ns.pop('cmd')
verbosity, traceback_cm = process_args_by_key(
args,
argv,
processors[cmd],
['verbosity', 'traceback_cm'],
)
if cmd != 'parse':
# "verbosity" is sent to the commands, so we put it back.
args.verbosity = verbosity
return cmd, ns, verbosity, traceback_cm
def main(cmd, cmd_kwargs):
try:
run_cmd = COMMANDS[cmd][-1]
except KeyError:
raise ValueError(f'unsupported cmd {cmd!r}')
run_cmd(**cmd_kwargs)
if __name__ == '__main__':
cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
configure_logger(verbosity)
with traceback_cm:
main(cmd, cmd_kwargs)
|