1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
|
#!/usr/bin/env python3
# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Extract source file information from .ninja files."""
# Copied from //tools/binary_size/libsupersize
import argparse
import io
import json
import logging
import os
import re
import sys
sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
import action_helpers
# E.g.:
# build obj/.../foo.o: cxx gen/.../foo.cc || obj/.../foo.inputdeps.stamp
# build obj/.../libfoo.a: alink obj/.../a.o obj/.../b.o |
# build ./libchrome.so ./lib.unstripped/libchrome.so: solink a.o b.o ...
# build libmonochrome.so: __chrome_android_libmonochrome___rule | ...
_REGEX = re.compile(r'build ([^:]+): \w+ (.*?)(?: *\||\n|$)')
_RLIBS_REGEX = re.compile(r' rlibs = (.*?)(?:\n|$)')
# Unmatches seems to happen for empty source_sets(). E.g.:
# obj/chrome/browser/ui/libui_public_dependencies.a
_MAX_UNMATCHED_TO_LOG = 20
_MAX_UNMATCHED_TO_IGNORE = 200
class _SourceMapper:
def __init__(self, dep_map, parsed_files):
self._dep_map = dep_map
self.parsed_files = parsed_files
self._unmatched_paths = set()
def _FindSourceForPathInternal(self, path):
if not path.endswith(')'):
if path.startswith('..'):
return path
return self._dep_map.get(path)
# foo/bar.a(baz.o)
start_idx = path.index('(')
lib_name = path[:start_idx]
obj_name = path[start_idx + 1:-1]
by_basename = self._dep_map.get(lib_name)
if not by_basename:
if lib_name.endswith('rlib') and 'std/' in lib_name:
# Currently we use binary prebuilt static libraries of the Rust
# stdlib so we can't get source paths. That may change in future.
return '(Rust stdlib)/%s' % lib_name
return None
if lib_name.endswith('.rlib'):
# Rust doesn't really have the concept of an object file because
# the compilation unit is the whole 'crate'. Return whichever
# filename was the crate root.
return next(iter(by_basename.values()))
obj_path = by_basename.get(obj_name)
if not obj_path:
# Found the library, but it doesn't list the .o file.
logging.warning('no obj basename for %s %s', path, obj_name)
return None
return self._dep_map.get(obj_path)
def FindSourceForPath(self, path):
"""Returns the source path for the given object path (or None if not found).
Paths for objects within archives should be in the format: foo/bar.a(baz.o)
"""
ret = self._FindSourceForPathInternal(path)
if not ret and path not in self._unmatched_paths:
if self.unmatched_paths_count < _MAX_UNMATCHED_TO_LOG:
logging.warning('Could not find source path for %s (empty source_set?)',
path)
self._unmatched_paths.add(path)
return ret
@property
def unmatched_paths_count(self):
return len(self._unmatched_paths)
def _ParseNinjaPathList(path_list):
ret = path_list.replace('\\ ', '\b')
return [s.replace('\b', ' ') for s in ret.split()]
def _OutputsAreObject(outputs):
return (outputs.endswith('.a') or outputs.endswith('.o')
or outputs.endswith('.rlib'))
def _ParseOneFile(lines, dep_map, executable_path):
sub_ninjas = []
executable_inputs = None
last_executable_paths = []
for line in lines:
if line.startswith('subninja '):
sub_ninjas.append(line[9:-1])
# Rust .rlibs are listed as implicit dependencies of the main
# target linking rule, then are given as an extra
# rlibs =
# variable on a subsequent line. Watch out for that line.
elif m := _RLIBS_REGEX.match(line):
if executable_path in last_executable_paths:
executable_inputs.extend(_ParseNinjaPathList(m.group(1)))
elif m := _REGEX.match(line):
outputs, srcs = m.groups()
if _OutputsAreObject(outputs):
output = outputs.replace('\\ ', ' ')
assert output not in dep_map, 'Duplicate output: ' + output
if output[-1] == 'o':
dep_map[output] = srcs.replace('\\ ', ' ')
else:
obj_paths = _ParseNinjaPathList(srcs)
dep_map[output] = {os.path.basename(p): p for p in obj_paths}
elif executable_path:
last_executable_paths = [
os.path.normpath(p) for p in _ParseNinjaPathList(outputs)
]
if executable_path in last_executable_paths:
executable_inputs = _ParseNinjaPathList(srcs)
return sub_ninjas, executable_inputs
def _Parse(output_directory, executable_path):
"""Parses build.ninja and subninjas.
Args:
output_directory: Where to find the root build.ninja.
executable_path: Path to binary to find inputs for.
Returns: A tuple of (source_mapper, executable_inputs).
source_mapper: _SourceMapper instance.
executable_inputs: List of paths of linker inputs.
"""
if executable_path:
executable_path = os.path.relpath(executable_path, output_directory)
to_parse = ['build.ninja']
seen_paths = set(to_parse)
dep_map = {}
executable_inputs = None
while to_parse:
path = os.path.join(output_directory, to_parse.pop())
with open(path, encoding='utf-8', errors='ignore') as obj:
sub_ninjas, found_executable_inputs = _ParseOneFile(
obj, dep_map, executable_path)
if found_executable_inputs:
assert not executable_inputs, (
'Found multiple inputs for executable_path ' + executable_path)
executable_inputs = found_executable_inputs
for subpath in sub_ninjas:
assert subpath not in seen_paths, 'Double include of ' + subpath
seen_paths.add(subpath)
to_parse.extend(sub_ninjas)
assert executable_inputs, 'Failed to find rule that builds ' + executable_path
return _SourceMapper(dep_map, seen_paths), executable_inputs
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--executable', required=True)
parser.add_argument('--result-json', required=True)
parser.add_argument('--depfile')
args = parser.parse_args()
logs_io = io.StringIO()
logging.basicConfig(level=logging.DEBUG,
format='%(levelname).1s %(relativeCreated)6d %(message)s',
stream=logs_io)
source_mapper, object_paths = _Parse('.', args.executable)
logging.info('Found %d linker inputs', len(object_paths))
source_paths = []
for obj_path in object_paths:
result = source_mapper.FindSourceForPath(obj_path) or obj_path
# Need to recurse on .a files.
if isinstance(result, dict):
source_paths.extend(
source_mapper.FindSourceForPath(v) or v for v in result.values())
else:
source_paths.append(result)
logging.info('Found %d source paths', len(source_paths))
num_unmatched = source_mapper.unmatched_paths_count
if num_unmatched > _MAX_UNMATCHED_TO_LOG:
logging.warning('%d paths were missing sources (showed the first %d)',
num_unmatched, _MAX_UNMATCHED_TO_LOG)
if num_unmatched > _MAX_UNMATCHED_TO_IGNORE:
raise Exception('Too many unmapped files. Likely a bug in ninja_parser.py')
if args.depfile:
action_helpers.write_depfile(args.depfile, args.result_json,
source_mapper.parsed_files)
with open(args.result_json, 'w', encoding='utf-8') as f:
json.dump({
'logs': logs_io.getvalue(),
'source_paths': source_paths,
}, f)
if __name__ == '__main__':
main()
|