#!/usr/bin/env python3 # # Copyright (C) 2019 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Call cargo -v, parse its output, and generate Android.bp. Usage: Run this script in a crate workspace root directory. The Cargo.toml file should work at least for the host platform. (1) Without other flags, "cargo2android.py --run" calls cargo clean, calls cargo build -v, and generates Android.bp. The cargo build only generates crates for the host, without test crates. (2) To build crates for both host and device in Android.bp, use the --device flag, for example: cargo2android.py --run --device Note that cargo build is only called once with the default target x86_64-unknown-linux-gnu. (3) To build default and test crates, for host and device, use both --device and --tests flags: cargo2android.py --run --device --tests This is equivalent to using the --cargo flag to add extra builds: cargo2android.py --run --cargo "build --target x86_64-unknown-linux-gnu" --cargo "build --tests --target x86_64-unknown-linux-gnu" (4) To build variants of the same crate include a list of variant entries under the 'variants' key inside cargo2android.json config file. Each entry should contain a 'suffix' key along with any required keys. The 'suffix' key must be a unique suffix to be concatenated to the stem/module name. Note that keys specified inside each variant entry will overwrite the values on the existing keys in cargo2android.json. If there are rustc warning messages, this script will add a warning comment to the owner crate module in Android.bp. """ from __future__ import annotations import argparse import glob import json import os import os.path import platform import re import shutil import subprocess import sys # Some Rust packages include extra unwanted crates. # This set contains all such excluded crate names. EXCLUDED_CRATES = {'protobuf_bin_gen_rust_do_not_use'} RENAME_MAP = { # This map includes all changes to the default rust module names # to resolve name conflicts, avoid confusion, or work as plugin. 'libash': 'libash_rust', 'libatomic': 'libatomic_rust', 'libbacktrace': 'libbacktrace_rust', 'libbase': 'libbase_rust', 'libbase64': 'libbase64_rust', 'libfuse': 'libfuse_rust', 'libgcc': 'libgcc_rust', 'liblog': 'liblog_rust', 'libminijail': 'libminijail_rust', 'libsync': 'libsync_rust', 'libx86_64': 'libx86_64_rust', 'libxml': 'libxml_rust', 'protoc_gen_rust': 'protoc-gen-rust', } RENAME_STEM_MAP = { # This map includes all changes to the default rust module stem names, # which is used for output files when different from the module name. 'protoc_gen_rust': 'protoc-gen-rust', } RENAME_DEFAULTS_MAP = { # This map includes all changes to the default prefix of rust_default # module names, to avoid conflict with existing Android modules. 'libc': 'rust_libc', } # Header added to all generated Android.bp files. ANDROID_BP_HEADER = ( '// This file is generated by cargo2android.py {args}.\n' + '// Do not modify this file as changes will be overridden on upgrade.\n') CARGO_OUT = 'cargo.out' # Name of file to keep cargo build -v output. # This should be kept in sync with tools/external_updater/crates_updater.py. ERRORS_LINE = 'Errors in ' + CARGO_OUT + ':' TARGET_TMP = 'target.tmp' # Name of temporary output directory. # Message to be displayed when this script is called without the --run flag. DRY_RUN_NOTE = ( 'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' + 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' + 'and writes to Android.bp in the current and subdirectories.\n\n' + 'To do do all of the above, use the --run flag.\n' + 'See --help for other flags, and more usage notes in this script.\n') # Cargo -v output of a call to rustc. RUSTC_PAT = re.compile('^ +Running `(.*\/)?rustc (.*)`$') # Cargo -vv output of a call to rustc could be split into multiple lines. # Assume that the first line will contain some CARGO_* env definition. RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$') # The combined -vv output rustc command line pattern. RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* (.*\/)?rustc (.*)`$') # Cargo -vv output of a "cc" or "ar" command; all in one line. CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$') # Some package, such as ring-0.13.5, has pattern '... running "cc"'. # Rustc output of file location path pattern for a warning message. WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+') # cargo test --list output of the start of running a binary. CARGO_TEST_LIST_START_PAT = re.compile('^\s*Running (.*) \(.*\)$') # cargo test --list output of the end of running a binary. CARGO_TEST_LIST_END_PAT = re.compile('^(\d+) tests?, (\d+) benchmarks$') CARGO2ANDROID_RUNNING_PAT = re.compile('^### Running: .*$') # Rust package name with suffix -d1.d2.d3(+.*)?. VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:-(alpha|beta)\.[0-9]+)?(?:\+.*)?$') # Crate types corresponding to a C ABI library C_LIBRARY_CRATE_TYPES = ['staticlib', 'cdylib'] # Crate types corresponding to a Rust ABI library RUST_LIBRARY_CRATE_TYPES = ['lib', 'rlib', 'dylib'] # Crate types corresponding to a library LIBRARY_CRATE_TYPES = C_LIBRARY_CRATE_TYPES + RUST_LIBRARY_CRATE_TYPES def altered_name(name): return RENAME_MAP[name] if (name in RENAME_MAP) else name def altered_stem(name): return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name def altered_defaults(name): return RENAME_DEFAULTS_MAP[name] if (name in RENAME_DEFAULTS_MAP) else name def is_build_crate_name(name): # We added special prefix to build script crate names. return name.startswith('build_script_') def is_dependent_file_path(path): # Absolute or dependent '.../' paths are not main files of this crate. return path.startswith('/') or path.startswith('.../') def get_module_name(crate): # to sort crates in a list return crate.module_name def pkg2crate_name(s): return s.replace('-', '_').replace('.', '_') def file_base_name(path): return os.path.splitext(os.path.basename(path))[0] def test_base_name(path): return pkg2crate_name(file_base_name(path)) def unquote(s): # remove quotes around str if s and len(s) > 1 and s[0] == s[-1] and s[0] in ('"', "'"): return s[1:-1] return s def remove_version_suffix(s): # remove -d1.d2.d3 suffix if VERSION_SUFFIX_PAT.match(s): return VERSION_SUFFIX_PAT.match(s).group(1) return s def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/* return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s) def escape_quotes(s): # replace '"' with '\\"' return s.replace('"', '\\"') class Crate(object): """Information of a Rust crate to collect/emit for an Android.bp module.""" def __init__(self, runner: Runner, outf_name): # Remembered global runner and its members. self.runner: Runner = runner self.debug = runner.args.debug self.cargo_dir = '' # directory of my Cargo.toml self.outf_name = outf_name # path to Android.bp self.outf = None # open file handle of outf_name during dump* # Variants/results that could be merged from multiple rustc lines. self.host_supported = False self.device_supported = False self.has_warning = False # Android module properties derived from rustc parameters. self.module_name = '' # unique in Android build system self.module_type = '' # rust_{binary,library,test}[_host] etc. self.defaults = '' # rust_defaults used by rust_test* modules self.default_srcs = False # use 'srcs' defined in self.defaults self.root_pkg = '' # parent package name of a sub/test packge, from -L self.srcs = list() # main_src or merged multiple source files self.stem = '' # real base name of output file # Kept parsed status self.errors = '' # all errors found during parsing self.line_num = 1 # runner told input source line number self.line = '' # original rustc command line parameters # Parameters collected from rustc command line. self.crate_name = '' # follows --crate-name self.main_src = '' # follows crate_name parameter, shortened self.crate_types = list() # follows --crate-type self.cfgs = list() # follows --cfg, without feature= prefix self.features = list() # follows --cfg, name in 'feature="..."' self.codegens = list() # follows -C, some ignored self.externs = list() # follows --extern self.core_externs = list() # first part of self.externs elements self.static_libs = list() # e.g. -l static=host_cpuid self.shared_libs = list() # e.g. -l dylib=wayland-client, -l z self.cap_lints = '' # follows --cap-lints self.emit_list = '' # e.g., --emit=dep-info,metadata,link self.edition = '2015' # rustc default, e.g., --edition=2018 self.target = '' # follows --target self.cargo_env_compat = True self.cargo_pkg_version = '' # value extracted from Cargo.toml version field self.variant_num = int(runner.variant_num) def write(self, s): # convenient way to output one line at a time with EOL. self.outf.write(s + '\n') def same_flags(self, other: Crate): # host_supported, device_supported, has_warning are not compared but merged # target is not compared, to merge different target/host modules # externs is not compared; only core_externs is compared return (not self.errors and not other.errors and self.edition == other.edition and self.cap_lints == other.cap_lints and self.emit_list == other.emit_list and self.core_externs == other.core_externs and self.codegens == other.codegens and self.features == other.features and self.static_libs == other.static_libs and self.shared_libs == other.shared_libs and self.cfgs == other.cfgs) def merge_host_device(self, other: Crate): """Returns true if attributes are the same except host/device support.""" return (self.crate_name == other.crate_name and self.crate_types == other.crate_types and self.main_src == other.main_src and # before merge, each test module has an unique module name and stem (self.stem == other.stem or self.crate_types == ['test']) and self.root_pkg == other.root_pkg and not self.skip_crate() and self.same_flags(other)) def merge_test(self, other: Crate): """Returns true if self and other are tests of same root_pkg.""" # Before merger, each test has its own crate_name. # A merged test uses its source file base name as output file name, # so a test is mergeable only if its base name equals to its crate name. return (self.crate_types == other.crate_types and self.crate_types == ['test'] and self.root_pkg == other.root_pkg and not self.skip_crate() and other.crate_name == test_base_name(other.main_src) and (len(self.srcs) > 1 or (self.crate_name == test_base_name(self.main_src)) and self.host_supported == other.host_supported and self.device_supported == other.device_supported) and self.same_flags(other)) def merge(self, other: Crate, outf_name): """Try to merge crate into self.""" # Cargo build --tests could recompile a library for tests. # We need to merge such duplicated calls to rustc, with # the algorithm in merge_host_device. should_merge_host_device = self.merge_host_device(other) should_merge_test = False if not should_merge_host_device: should_merge_test = self.merge_test(other) if should_merge_host_device or should_merge_test: self.runner.init_bp_file(outf_name) with open(outf_name, 'a') as outf: # to write debug info self.outf = outf other.outf = outf self.do_merge(other, should_merge_test) return True return False def do_merge(self, other: Crate, should_merge_test): """Merge attributes of other to self.""" if self.debug: self.write('\n// Before merge definition (1):') self.dump_debug_info() self.write('\n// Before merge definition (2):') other.dump_debug_info() # Merge properties of other to self. self.has_warning = self.has_warning or other.has_warning if not self.target: # okay to keep only the first target triple self.target = other.target # decide_module_type sets up default self.stem, # which can be changed if self is a merged test module. self.decide_module_type() if should_merge_test: if (self.runner.should_ignore_test(self.main_src) and not self.runner.should_ignore_test(other.main_src)): self.main_src = other.main_src self.srcs.append(other.main_src) # use a short unique name as the merged module name. prefix = self.root_pkg + '_tests' self.module_name = self.runner.claim_module_name(prefix, self, 0) + self.runner.args.name_suffix self.stem = self.module_name # This normalized root_pkg name although might be the same # as other module's crate_name, it is not actually used for # output file name. A merged test module always have multiple # source files and each source file base name is used as # its output file name. self.crate_name = pkg2crate_name(self.root_pkg) if self.debug: self.write('\n// After merge definition (1):') self.dump_debug_info() def find_cargo_dir(self): """Deepest directory with Cargo.toml and contains the main_src.""" if not is_dependent_file_path(self.main_src): dir_name = os.path.dirname(self.main_src) while dir_name: if os.path.exists(dir_name + '/Cargo.toml'): self.cargo_dir = dir_name return dir_name = os.path.dirname(dir_name) def add_codegens_flag(self, flag: str): """Ignore options not used in Android.""" # 'prefer-dynamic' does not work with common flag -C lto # 'embed-bitcode' is ignored; we might control LTO with other .bp flag # 'codegen-units' is set in Android global config or by default if not (flag.startswith('codegen-units=') or flag.startswith('debuginfo=') or flag.startswith('embed-bitcode=') or flag.startswith('extra-filename=') or flag.startswith('incremental=') or flag.startswith('metadata=') or flag == 'prefer-dynamic'): self.codegens.append(flag) def parse(self, line_num, line): """Find important rustc arguments to convert to Android.bp properties.""" self.line_num = line_num self.line = line args = list(map(unquote, line.split())) i = 0 # Loop through every argument of rustc. while i < len(args): arg = args[i] if arg == '--crate-name': i += 1 self.crate_name = args[i] elif arg == '--crate-type': i += 1 # cargo calls rustc with multiple --crate-type flags. # rustc can accept: # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] self.crate_types.append(args[i]) elif arg == '--test': self.crate_types.append('test') elif arg == '--target': i += 1 self.target = args[i] elif arg == '--cfg': i += 1 if args[i].startswith('feature='): self.features.append(unquote(args[i].replace('feature=', ''))) else: self.cfgs.append(args[i]) elif arg == '--extern': i += 1 extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i]) self.externs.append(extern_names) self.core_externs.append(re.sub(' = .*', '', extern_names)) elif arg == '-C': # codegen options i += 1 self.add_codegens_flag(args[i]) elif arg.startswith('-C'): # cargo has been passing "-C " flag to rustc, # but newer cargo could pass '-Cembed-bitcode=no' to rustc. self.add_codegens_flag(arg[2:]) elif arg == '--cap-lints': i += 1 self.cap_lints = args[i] elif arg == '-L': i += 1 if args[i].startswith('dependency=') and args[i].endswith('/deps'): if '/' + TARGET_TMP + '/' in args[i]: self.root_pkg = re.sub( '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i])) else: self.root_pkg = re.sub('^.*/', '', re.sub('/[^/]+/[^/]+/deps$', '', args[i])) self.root_pkg = remove_version_suffix(self.root_pkg) elif arg == '-l': i += 1 if args[i].startswith('static='): self.static_libs.append(re.sub('static=', '', args[i])) elif args[i].startswith('dylib='): self.shared_libs.append(re.sub('dylib=', '', args[i])) else: self.shared_libs.append(args[i]) elif arg == '--out-dir' or arg == '--color': # ignored i += 1 elif arg.startswith('--error-format=') or arg.startswith('--json='): pass # ignored elif arg.startswith('--emit='): self.emit_list = arg.replace('--emit=', '') elif arg.startswith('--edition='): self.edition = arg.replace('--edition=', '') elif arg.startswith('-Aclippy') or arg.startswith('-Wclippy'): pass # TODO: Consider storing these to include in the Android.bp. elif arg.startswith('-W'): pass # ignored elif arg.startswith('-D'): pass # TODO: Consider storing these to include in the Android.bp. elif not arg.startswith('-'): # shorten imported crate main source paths like $HOME/.cargo/ # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/lib.rs self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', args[i]) self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../', self.main_src) self.find_cargo_dir() if self.cargo_dir: # for a subdirectory if self.runner.variant_args.no_subdir: # all .bp content to /dev/null self.outf_name = '/dev/null' elif not self.runner.variant_args.onefile: # Write to Android.bp in the subdirectory with Cargo.toml. self.outf_name = self.cargo_dir + '/Android.bp' self.main_src = self.main_src[len(self.cargo_dir) + 1:] else: self.errors += 'ERROR: unknown ' + arg + '\n' i += 1 if not self.crate_name: self.errors += 'ERROR: missing --crate-name\n' if not self.main_src: self.errors += 'ERROR: missing main source file\n' else: self.srcs.append(self.main_src) if not self.crate_types: # Treat "--cfg test" as "--test" if 'test' in self.cfgs: self.crate_types.append('test') else: self.errors += 'ERROR: missing --crate-type or --test\n' elif len(self.crate_types) > 1: if 'test' in self.crate_types: self.errors += 'ERROR: cannot handle both --crate-type and --test\n' if 'lib' in self.crate_types and 'rlib' in self.crate_types: self.errors += 'ERROR: cannot generate both lib and rlib crate types\n' if not self.root_pkg: self.root_pkg = self.crate_name # get the package version from running cargo metadata if not self.runner.variant_args.no_pkg_vers and not self.skip_crate(): self.get_pkg_version() self.device_supported = self.runner.variant_args.device self.host_supported = not self.runner.variant_args.no_host self.cfgs = sorted(set(self.cfgs)) self.features = sorted(set(self.features)) self.codegens = sorted(set(self.codegens)) self.externs = sorted(set(self.externs)) self.core_externs = sorted(set(self.core_externs)) self.static_libs = sorted(set(self.static_libs)) self.shared_libs = sorted(set(self.shared_libs)) self.crate_types = sorted(set(self.crate_types)) self.decide_module_type() self.module_name = altered_name(self.stem) + self.runner.args.name_suffix return self def get_pkg_version(self): """Attempt to retrieve the package version from the Cargo.toml If there is only one package, use its version. Otherwise, try to match the emitted `--crate_name` arg against the package name. This may fail in cases where multiple packages are defined (workspaces) and where the package name does not match the emitted crate_name (e.g. [lib.name] is set). """ cargo_metadata = subprocess.run([self.runner.cargo_path, 'metadata', '--no-deps', '--format-version', '1'], cwd=os.path.abspath(self.cargo_dir), stdout=subprocess.PIPE) if cargo_metadata.returncode: self.errors += ('ERROR: unable to get cargo metadata for package version; ' + 'return code ' + cargo_metadata.returncode + '\n') else: metadata_json = json.loads(cargo_metadata.stdout) if len(metadata_json['packages']) > 1: for package in metadata_json['packages']: # package names may contain '-', but is changed to '_' in the crate_name if package['name'].replace('-','_') == self.crate_name: self.cargo_pkg_version = package['version'] break else: self.cargo_pkg_version = metadata_json['packages'][0]['version'] if not self.cargo_pkg_version: self.errors += ('ERROR: Unable to retrieve package version; ' + 'to disable, run with arg "--no-pkg-vers"\n') def dump_line(self): self.write('\n// Line ' + str(self.line_num) + ' ' + self.line) def feature_list(self): """Return a string of main_src + "feature_list".""" pkg = self.main_src if pkg.startswith('.../'): # keep only the main package name pkg = re.sub('/.*', '', pkg[4:]) elif pkg.startswith('/'): # use relative path for a local package pkg = os.path.relpath(pkg) if not self.features: return pkg return pkg + ' "' + ','.join(self.features) + '"' def dump_skip_crate(self, kind): if self.debug: self.write('\n// IGNORED: ' + kind + ' ' + self.main_src) return self def skip_crate(self): """Return crate_name or a message if this crate should be skipped.""" if (is_build_crate_name(self.crate_name) or self.crate_name in EXCLUDED_CRATES): return self.crate_name if is_dependent_file_path(self.main_src): return 'dependent crate' return '' def dump(self): """Dump all error/debug/module code to the output .bp file.""" self.runner.init_bp_file(self.outf_name) with open(self.outf_name, 'a') as outf: self.outf = outf if self.errors: self.dump_line() self.write(self.errors) elif self.skip_crate(): self.dump_skip_crate(self.skip_crate()) else: if self.debug: self.dump_debug_info() self.dump_android_module() def dump_debug_info(self): """Dump parsed data, when cargo2android is called with --debug.""" def dump(name, value): self.write('//%12s = %s' % (name, value)) def opt_dump(name, value): if value: dump(name, value) def dump_list(fmt, values): for v in values: self.write(fmt % v) self.dump_line() dump('module_name', self.module_name) dump('crate_name', self.crate_name) dump('crate_types', self.crate_types) dump('main_src', self.main_src) dump('has_warning', self.has_warning) dump('for_host', self.host_supported) dump('for_device', self.device_supported) dump('module_type', self.module_type) opt_dump('target', self.target) opt_dump('edition', self.edition) opt_dump('emit_list', self.emit_list) opt_dump('cap_lints', self.cap_lints) dump_list('// cfg = %s', self.cfgs) dump_list('// cfg = \'feature "%s"\'', self.features) # TODO(chh): escape quotes in self.features, but not in other dump_list dump_list('// codegen = %s', self.codegens) dump_list('// externs = %s', self.externs) dump_list('// -l static = %s', self.static_libs) dump_list('// -l (dylib) = %s', self.shared_libs) def dump_android_module(self): """Dump one or more Android module definition, depending on crate_types.""" if len(self.crate_types) == 1: self.dump_single_type_android_module() return if 'test' in self.crate_types: self.write('\nERROR: multiple crate types cannot include test type') return # Dump one Android module per crate_type. for crate_type in self.crate_types: self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) def build_default_name(self): """Return a short and readable name for the rust_defaults module.""" # Choices: (1) root_pkg + '_test'? + '_defaults', # (2) root_pkg + '_test'? + '_defaults_' + crate_name # (3) root_pkg + '_test'? + '_defaults_' + main_src_basename_path # (4) root_pkg + '_test'? + '_defaults_' + a_positive_sequence_number test = "_test" if self.crate_types == ['test'] else "" name1 = altered_defaults(self.root_pkg) + test + '_defaults' if self.runner.try_claim_module_name(name1, self): return name1 name2 = name1 + '_' + self.crate_name if self.runner.try_claim_module_name(name2, self): return name2 name3 = name1 + '_' + self.main_src_basename_path() if self.runner.try_claim_module_name(name3, self): return name3 return self.runner.claim_module_name(name1, self, 0) def dump_srcs_list(self): """Dump the srcs list, for defaults or regular modules.""" if len(self.srcs) > 1: srcs = sorted(set(self.srcs)) # make a copy and dedup else: srcs = [self.main_src] copy_out = self.runner.copy_out_module_name() if copy_out: srcs.append(':' + copy_out) self.dump_android_property_list('srcs', '"%s"', srcs) def dump_defaults_module(self): """Dump a rust_defaults module to be shared by other modules.""" name = self.build_default_name() self.defaults = name self.write('\nrust_defaults {') self.write(' name: "' + name + '",') if self.runner.variant_args.global_defaults: self.write(' defaults: ["' + self.runner.variant_args.global_defaults + '"],') self.write(' crate_name: "' + self.crate_name + '",') if len(self.srcs) == 1: # only one source file; share it in defaults self.default_srcs = True if self.has_warning and not self.cap_lints: self.write(' // has rustc warnings') self.dump_srcs_list() if self.cargo_env_compat: self.write(' cargo_env_compat: true,') if not self.runner.variant_args.no_pkg_vers: self.write(' cargo_pkg_version: "' + self.cargo_pkg_version + '",') if 'test' in self.crate_types: self.write(' test_suites: ["general-tests"],') self.write(' auto_gen_config: true,') self.dump_edition_flags_libs() if 'test' in self.crate_types and len(self.srcs) == 1: self.dump_test_data() self.write('}') def dump_single_type_android_module(self): """Dump one simple Android module, which has only one crate_type.""" crate_type = self.crate_types[0] if crate_type != 'test': # do not change self.stem or self.module_name self.dump_one_android_module(crate_type) return # Dump one test module per source file. # crate_type == 'test' self.srcs = [src for src in self.srcs if not self.runner.should_ignore_test(src)] if len(self.srcs) > 1: self.srcs = sorted(set(self.srcs)) self.dump_defaults_module() saved_srcs = self.srcs for src in saved_srcs: self.srcs = [src] saved_main_src = self.main_src self.main_src = src self.module_name = self.test_module_name() self.decide_one_module_type(crate_type) self.dump_one_android_module(crate_type) self.main_src = saved_main_src self.srcs = saved_srcs def dump_one_android_module(self, crate_type): """Dump one Android module definition.""" if not self.module_type: self.write('\nERROR: unknown crate_type ' + crate_type) return self.write('\n' + self.module_type + ' {') self.dump_android_core_properties() if not self.defaults: self.dump_edition_flags_libs() if self.runner.variant_args.host_first_multilib and self.host_supported and crate_type != 'test': self.write(' compile_multilib: "first",') if self.runner.variant_args.exported_c_header_dir and crate_type in C_LIBRARY_CRATE_TYPES: self.write(' include_dirs: [') for header_dir in self.runner.variant_args.exported_c_header_dir: self.write(' "%s",' % header_dir) self.write(' ],') if crate_type in LIBRARY_CRATE_TYPES and self.device_supported: self.write(' apex_available: [') if self.runner.variant_args.apex_available is None: # If apex_available is not explicitly set, make it available to all # apexes. self.write(' "//apex_available:platform",') self.write(' "//apex_available:anyapex",') else: for apex in self.runner.variant_args.apex_available: self.write(' "%s",' % apex) self.write(' ],') if crate_type != 'test': if self.runner.variant_args.no_std: self.write(' prefer_rlib: true,') self.write(' no_stdlibs: true,') self.write(' stdlibs: [') if self.runner.variant_args.alloc: self.write(' "liballoc.rust_sysroot",') self.write(' "libcompiler_builtins.rust_sysroot",') self.write(' "libcore.rust_sysroot",') self.write(' ],') if self.device_supported: # These configurations are meaningful only if it is for device. if self.runner.variant_args.native_bridge_supported: self.write(' native_bridge_supported: true,') if self.runner.variant_args.product_available: self.write(' product_available: true,') if self.runner.variant_args.recovery_available: self.write(' recovery_available: true,') if self.runner.variant_args.vendor_available: self.write(' vendor_available: true,') if self.runner.variant_args.vendor_ramdisk_available: self.write(' vendor_ramdisk_available: true,') if self.runner.variant_args.ramdisk_available: self.write(' ramdisk_available: true,') if self.runner.variant_args.min_sdk_version and crate_type in LIBRARY_CRATE_TYPES and self.device_supported: self.write(' min_sdk_version: "%s",' % self.runner.variant_args.min_sdk_version) if crate_type == 'test' and not self.default_srcs: self.dump_test_data() if self.runner.variant_args.add_module_block: with open(self.runner.variant_args.add_module_block, 'r') as f: self.write(' %s,' % f.read().replace('\n', '\n ')) self.write('}') def dump_android_flags(self): """Dump Android module flags property.""" if not self.codegens and not self.cap_lints: return self.write(' flags: [') if self.cap_lints: self.write(' "--cap-lints ' + self.cap_lints + '",') codegens_fmt = '"-C %s"' self.dump_android_property_list_items(codegens_fmt, self.codegens) self.write(' ],') def dump_edition_flags_libs(self): if self.edition: self.write(' edition: "' + self.edition + '",') self.dump_android_property_list('features', '"%s"', self.features) cfgs = [cfg for cfg in self.cfgs if not cfg in self.runner.variant_args.cfg_blocklist] self.dump_android_property_list('cfgs', '"%s"', cfgs) self.dump_android_flags() if self.externs: self.dump_android_externs() all_static_libs = [lib for lib in self.static_libs if not lib in self.runner.variant_args.lib_blocklist] static_libs = [lib for lib in all_static_libs if not lib in self.runner.variant_args.whole_static_libs] self.dump_android_property_list('static_libs', '"lib%s"', static_libs) whole_static_libs = [lib for lib in all_static_libs if lib in self.runner.variant_args.whole_static_libs] self.dump_android_property_list('whole_static_libs', '"lib%s"', whole_static_libs) shared_libs = [lib for lib in self.shared_libs if not lib in self.runner.variant_args.lib_blocklist] self.dump_android_property_list('shared_libs', '"lib%s"', shared_libs) def dump_test_data(self): data = [data for (name, data) in map(lambda kv: kv.split('=', 1), self.runner.variant_args.test_data) if self.srcs == [name]] if data: self.dump_android_property_list('data', '"%s"', data) def main_src_basename_path(self): return re.sub('/', '_', re.sub('.rs$', '', self.main_src)) def test_module_name(self): """Return a unique name for a test module.""" # root_pkg+(_host|_device) + '_test_'+source_file_name suffix = self.main_src_basename_path() return self.root_pkg + '_test_' + suffix def decide_module_type(self): # Use the first crate type for the default/first module. crate_type = self.crate_types[0] if self.crate_types else '' self.decide_one_module_type(crate_type) def decide_one_module_type(self, crate_type): """Decide which Android module type to use.""" host = '' if self.device_supported else '_host' rlib = '_rlib' if self.runner.variant_args.force_rlib else '' suffix = self.runner.variant_args.suffix if 'suffix' in self.runner.variant_args else '' if crate_type == 'bin': # rust_binary[_host] self.module_type = 'rust_binary' + host # In rare cases like protobuf-codegen, the output binary name must # be renamed to use as a plugin for protoc. self.stem = altered_stem(self.crate_name) + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) elif crate_type == 'lib': # rust_library[_host] # TODO(chh): should this be rust_library[_host]? # Assuming that Cargo.toml do not use both 'lib' and 'rlib', # because we map them both to rlib. self.module_type = 'rust_library' + host + rlib self.stem = 'lib' + self.crate_name + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) elif crate_type == 'rlib': # rust_library[_host] self.module_type = 'rust_library' + host + rlib self.stem = 'lib' + self.crate_name + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) elif crate_type == 'dylib': # rust_library[_host]_dylib self.module_type = 'rust_library' + host + '_dylib' self.stem = 'lib' + self.crate_name + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) + '_dylib' elif crate_type == 'cdylib': # rust_library[_host]_shared self.module_type = 'rust_ffi' + host + '_shared' self.stem = 'lib' + self.crate_name + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) + '_shared' elif crate_type == 'staticlib': # rust_library[_host]_static self.module_type = 'rust_ffi' + host + '_static' self.stem = 'lib' + self.crate_name + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) + '_static' elif crate_type == 'test': # rust_test[_host] self.module_type = 'rust_test' + host # Before do_merge, stem name is based on the --crate-name parameter. # and test module name is based on stem. self.stem = self.test_module_name() # self.stem will be changed after merging with other tests. # self.stem is NOT used for final test binary name. # rust_test uses each source file base name as part of output file name. # In do_merge, this function is called again, with a module_name. # We make sure that the module name is unique in each package. if self.module_name: # Cargo uses "-C extra-filename=..." and "-C metadata=..." to add # different suffixes and distinguish multiple tests of the same # crate name. We ignore -C and use claim_module_name to get # unique sequential suffix. self.module_name = self.runner.claim_module_name( self.module_name, self, 0) + self.runner.args.name_suffix # Now the module name is unique, stem should also match and unique. self.stem = self.module_name elif crate_type == 'proc-macro': # rust_proc_macro self.module_type = 'rust_proc_macro' self.stem = 'lib' + self.crate_name + suffix + self.runner.args.name_suffix self.module_name = altered_name(self.stem) else: # unknown module type, rust_prebuilt_dylib? rust_library[_host]? self.module_type = '' self.stem = '' def dump_android_property_list_items(self, fmt, values): for v in values: # fmt has quotes, so we need escape_quotes(v) self.write(' ' + (fmt % escape_quotes(v)) + ',') def dump_android_property_list(self, name, fmt, values): if not values: return if len(values) > 1: self.write(' ' + name + ': [') self.dump_android_property_list_items(fmt, values) self.write(' ],') else: self.write(' ' + name + ': [' + (fmt % escape_quotes(values[0])) + '],') def dump_android_core_properties(self): """Dump the module header, name, stem, etc.""" self.write(' name: "' + self.module_name + '",') # see properties shared by dump_defaults_module if self.defaults: self.write(' defaults: ["' + self.defaults + '"],') elif self.runner.variant_args.global_defaults: self.write(' defaults: ["' + self.runner.variant_args.global_defaults + '"],') if self.stem != self.module_name: self.write(' stem: "' + self.stem + '",') if self.has_warning and not self.cap_lints and not self.default_srcs: self.write(' // has rustc warnings') if self.host_supported and self.device_supported and self.module_type != 'rust_proc_macro': self.write(' host_supported: true,') if not self.defaults: self.write(' crate_name: "' + self.crate_name + '",') if not self.defaults and self.cargo_env_compat: self.write(' cargo_env_compat: true,') if not self.runner.variant_args.no_pkg_vers: self.write(' cargo_pkg_version: "' + self.cargo_pkg_version + '",') if not self.default_srcs: self.dump_srcs_list() if 'test' in self.crate_types and not self.defaults: # self.root_pkg can have multiple test modules, with different *_tests[n] # names, but their executables can all be installed under the same _tests # directory. When built from Cargo.toml, all tests should have different # file or crate names. So we used (root_pkg + '_tests') name as the # relative_install_path. # However, some package like 'slab' can have non-mergeable tests that # must be separated by different module names. So, here we no longer # emit relative_install_path. # self.write(' relative_install_path: "' + self.root_pkg + '_tests",') self.write(' test_suites: ["general-tests"],') self.write(' auto_gen_config: true,') if 'test' in self.crate_types and self.host_supported: self.write(' test_options: {') if self.runner.variant_args.no_presubmit: self.write(' unit_test: false,') else: self.write(' unit_test: true,') self.write(' },') def dump_android_externs(self): """Dump the dependent rlibs and dylibs property.""" so_libs = list() rust_libs = '' deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$') dependency_suffix = self.runner.variant_args.dependency_suffix or '' for lib in self.externs: # normal value of lib: "libc = liblibc-*.rlib" # strange case in rand crate: "getrandom_package = libgetrandom-*.rlib" # we should use "libgetrandom", not "lib" + "getrandom_package" groups = deps_libname.match(lib) if groups is not None: lib_name = groups.group(1) else: lib_name = re.sub(' .*$', '', lib) if lib_name in self.runner.variant_args.dependency_blocklist: continue if lib_name in self.runner.args.dep_suffixes: lib_name += self.runner.args.dep_suffixes[lib_name] if lib.endswith('.rlib') or lib.endswith('.rmeta'): # On MacOS .rmeta is used when Linux uses .rlib or .rmeta. rust_libs += ' "' + altered_name('lib' + lib_name + dependency_suffix) + '",\n' elif lib.endswith('.so'): so_libs.append(lib_name) elif lib != 'proc_macro': # --extern proc_macro is special and ignored rust_libs += ' // ERROR: unknown type of lib ' + lib + '\n' if rust_libs: self.write(' rustlibs: [\n' + rust_libs + ' ],') # Are all dependent .so files proc_macros? # TODO(chh): Separate proc_macros and dylib. self.dump_android_property_list('proc_macros', '"lib%s"', so_libs) class ARObject(object): """Information of an "ar" link command.""" def __init__(self, runner, outf_name): # Remembered global runner and its members. self.runner = runner self.pkg = '' self.outf_name = outf_name # path to Android.bp # "ar" arguments self.line_num = 1 self.line = '' self.flags = '' # e.g. "crs" self.lib = '' # e.g. "/.../out/lib*.a" self.objs = list() # e.g. "/.../out/.../*.o" def parse(self, pkg, line_num, args_line): """Collect ar obj/lib file names.""" self.pkg = pkg self.line_num = line_num self.line = args_line args = args_line.split() num_args = len(args) if num_args < 3: print('ERROR: "ar" command has too few arguments', args_line) else: self.flags = unquote(args[0]) self.lib = unquote(args[1]) self.objs = sorted(set(map(unquote, args[2:]))) return self def write(self, s): self.outf.write(s + '\n') def dump_debug_info(self): self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line) self.write('// ar_object for %12s' % self.pkg) self.write('// flags = %s' % self.flags) self.write('// lib = %s' % short_out_name(self.pkg, self.lib)) for o in self.objs: self.write('// obj = %s' % short_out_name(self.pkg, o)) def dump_android_lib(self): """Write cc_library_static into Android.bp.""" self.write('\ncc_library_static {') self.write(' name: "' + file_base_name(self.lib) + '",') self.write(' host_supported: true,') if self.flags != 'crs': self.write(' // ar flags = %s' % self.flags) if self.pkg not in self.runner.pkg_obj2cc: self.write(' ERROR: cannot find source files.\n}') return self.write(' srcs: [') obj2cc = self.runner.pkg_obj2cc[self.pkg] # Note: wflags are ignored. dflags = list() fflags = list() for obj in self.objs: self.write(' "' + short_out_name(self.pkg, obj2cc[obj].src) + '",') # TODO(chh): union of dflags and flags of all obj # Now, just a temporary hack that uses the last obj's flags dflags = obj2cc[obj].dflags fflags = obj2cc[obj].fflags self.write(' ],') self.write(' cflags: [') self.write(' "-O3",') # TODO(chh): is this default correct? self.write(' "-Wno-error",') for x in fflags: self.write(' "-f' + x + '",') for x in dflags: self.write(' "-D' + x + '",') self.write(' ],') self.write('}') def dump(self): """Dump error/debug/module info to the output .bp file.""" self.runner.init_bp_file(self.outf_name) with open(self.outf_name, 'a') as outf: self.outf = outf if self.runner.variant_args.debug: self.dump_debug_info() self.dump_android_lib() class CCObject(object): """Information of a "cc" compilation command.""" def __init__(self, runner, outf_name): # Remembered global runner and its members. self.runner = runner self.pkg = '' self.outf_name = outf_name # path to Android.bp # "cc" arguments self.line_num = 1 self.line = '' self.src = '' self.obj = '' self.dflags = list() # -D flags self.fflags = list() # -f flags self.iflags = list() # -I flags self.wflags = list() # -W flags self.other_args = list() def parse(self, pkg, line_num, args_line): """Collect cc compilation flags and src/out file names.""" self.pkg = pkg self.line_num = line_num self.line = args_line args = args_line.split() i = 0 while i < len(args): arg = args[i] if arg == '"-c"': i += 1 if args[i].startswith('"-o'): # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c" self.obj = unquote(args[i])[2:] i += 1 self.src = unquote(args[i]) else: self.src = unquote(args[i]) elif arg == '"-o"': i += 1 self.obj = unquote(args[i]) elif arg == '"-I"': i += 1 self.iflags.append(unquote(args[i])) elif arg.startswith('"-D'): self.dflags.append(unquote(args[i])[2:]) elif arg.startswith('"-f'): self.fflags.append(unquote(args[i])[2:]) elif arg.startswith('"-W'): self.wflags.append(unquote(args[i])[2:]) elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or arg == '"-g3"'): # ignore -O -m64 -g self.other_args.append(unquote(args[i])) i += 1 self.dflags = sorted(set(self.dflags)) self.fflags = sorted(set(self.fflags)) # self.wflags is not sorted because some are order sensitive # and we ignore them anyway. if self.pkg not in self.runner.pkg_obj2cc: self.runner.pkg_obj2cc[self.pkg] = {} self.runner.pkg_obj2cc[self.pkg][self.obj] = self return self def write(self, s): self.outf.write(s + '\n') def dump_debug_flags(self, name, flags): self.write('// ' + name + ':') for f in flags: self.write('// %s' % f) def dump(self): """Dump only error/debug info to the output .bp file.""" if not self.runner.variant_args.debug: return self.runner.init_bp_file(self.outf_name) with open(self.outf_name, 'a') as outf: self.outf = outf self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line) self.write('// cc_object for %12s' % self.pkg) self.write('// src = %s' % short_out_name(self.pkg, self.src)) self.write('// obj = %s' % short_out_name(self.pkg, self.obj)) self.dump_debug_flags('-I flags', self.iflags) self.dump_debug_flags('-D flags', self.dflags) self.dump_debug_flags('-f flags', self.fflags) self.dump_debug_flags('-W flags', self.wflags) if self.other_args: self.dump_debug_flags('other args', self.other_args) class Runner(object): """Main class to parse cargo -v output and print Android module definitions.""" def __init__(self, args: argparse.Namespace): self.bp_files = set() # Remember all output Android.bp files. self.root_pkg = '' # name of package in ./Cargo.toml # Saved flags, modes, and data. self.args = args self.variant_args = args self.variant_num = 0 self.dry_run = not args.run self.skip_cargo = args.skipcargo self.cargo_path = './cargo' # path to cargo, will be set later self.checked_out_files = False # to check only once self.build_out_files = [] # output files generated by build.rs # All cc/ar objects, crates, dependencies, and warning files self.cc_objects = list() self.pkg_obj2cc = {} # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i] self.ar_objects = list() self.crates: list[Crate] = list() self.warning_files = set() # Keep a unique mapping from (module name) to crate self.name_owners = {} # Save and dump all errors from cargo to Android.bp. self.errors = '' self.test_errors = '' self.setup_cargo_path() self.empty_tests = set() self.empty_unittests = False def setup_cargo_path(self): """Find cargo in the --cargo_bin or prebuilt rust bin directory.""" if self.args.cargo_bin: self.cargo_path = os.path.join(self.args.cargo_bin, 'cargo') if not os.path.isfile(self.cargo_path): sys.exit('ERROR: cannot find cargo in ' + self.args.cargo_bin) print('INFO: using cargo in ' + self.args.cargo_bin) return # We have only tested this on Linux. if platform.system() != 'Linux': sys.exit('ERROR: this script has only been tested on Linux with cargo.') # Assuming that this script is in development/scripts. my_dir = os.path.dirname(os.path.abspath(__file__)) linux_dir = os.path.join(my_dir, '..', '..', 'prebuilts', 'rust', 'linux-x86') if not os.path.isdir(linux_dir): sys.exit('ERROR: cannot find directory ' + linux_dir) rust_version = self.find_rust_version(my_dir, linux_dir) cargo_bin = os.path.join(linux_dir, rust_version, 'bin') self.cargo_path = os.path.join(cargo_bin, 'cargo') if not os.path.isfile(self.cargo_path): sys.exit('ERROR: cannot find cargo in ' + cargo_bin + '; please try --cargo_bin= flag.') return def find_rust_version(self, my_dir, linux_dir): """Use my script directory, find prebuilt rust version.""" # First look up build/soong/rust/config/global.go. path2global = os.path.join(my_dir, '..', '..', 'build', 'soong', 'rust', 'config', 'global.go') if os.path.isfile(path2global): # try to find: RustDefaultVersion = "1.44.0" version_pat = re.compile( r'\s*RustDefaultVersion\s*=\s*"([0-9]+\.[0-9]+\.[0-9]+).*"') with open(path2global, 'r') as inf: for line in inf: result = version_pat.match(line) if result: return result.group(1) print('WARNING: cannot find RustDefaultVersion in ' + path2global) # Otherwise, find the newest (largest) version number in linux_dir. rust_version = (0, 0, 0) # the prebuilt version to use version_pat = re.compile(r'([0-9]+)\.([0-9]+)\.([0-9]+)$') for dir_name in os.listdir(linux_dir): result = version_pat.match(dir_name) if not result: continue version = (int(result.group(1)), int(result.group(2)), int(result.group(3))) if version > rust_version: rust_version = version return '.'.join(map(str, rust_version)) def find_out_files(self): # list1 has build.rs output for normal crates list1 = glob.glob(TARGET_TMP + '/*/*/build/' + self.root_pkg + '-*/out/*') # list2 has build.rs output for proc-macro crates list2 = glob.glob(TARGET_TMP + '/*/build/' + self.root_pkg + '-*/out/*') return list1 + list2 def copy_out_files(self): """Copy build.rs output files to ./out and set up build_out_files.""" if self.checked_out_files: return self.checked_out_files = True cargo_out_files = self.find_out_files() out_files = set() if cargo_out_files: os.makedirs('out', exist_ok=True) for path in cargo_out_files: file_name = path.split('/')[-1] out_files.add(file_name) shutil.copy(path, 'out/' + file_name) self.build_out_files = sorted(out_files) def has_used_out_dir(self): """Returns true if env!("OUT_DIR") is found.""" return 0 == os.system('grep -rl --exclude build.rs --include \\*.rs' + ' \'env!("OUT_DIR")\' * > /dev/null') def copy_out_module_name(self): if self.args.copy_out and self.build_out_files: return 'copy_' + self.root_pkg + '_build_out' else: return '' def read_license(self, name): if not os.path.isfile(name): return '' license = '' with open(name, 'r') as intf: line = intf.readline() # Firstly skip ANDROID_BP_HEADER while line.startswith('//'): line = intf.readline() # Read all lines until we see a rust_* or genrule rule. while line != '' and not (line.startswith('rust_') or line.startswith('genrule {')): license += line line = intf.readline() return license.strip() def dump_copy_out_module(self, outf): """Output the genrule module to copy out/* to $(genDir).""" copy_out = self.copy_out_module_name() if not copy_out: return outf.write('\ngenrule {\n') outf.write(' name: "' + copy_out + '",\n') outf.write(' srcs: ["out/*"],\n') outf.write(' cmd: "cp $(in) $(genDir)",\n') if len(self.build_out_files) > 1: outf.write(' out: [\n') for f in self.build_out_files: outf.write(' "' + f + '",\n') outf.write(' ],\n') else: outf.write(' out: ["' + self.build_out_files[0] + '"],\n') outf.write('}\n') def init_bp_file(self, name): # name could be Android.bp or sub_dir_path/Android.bp if name not in self.bp_files: self.bp_files.add(name) license_section = self.read_license(name) with open(name, 'w') as outf: print_args = sys.argv[1:].copy() if '--cargo_bin' in print_args: index = print_args.index('--cargo_bin') del print_args[index:index+2] outf.write(ANDROID_BP_HEADER.format(args=' '.join(print_args))) outf.write('\n') outf.write(license_section) outf.write('\n') # at most one copy_out module per .bp file self.dump_copy_out_module(outf) def try_claim_module_name(self, name, owner): """Reserve and return True if it has not been reserved yet.""" if name not in self.name_owners or owner == self.name_owners[name]: self.name_owners[name] = owner return True return False def claim_module_name(self, prefix, owner, counter): """Return prefix if not owned yet, otherwise, prefix+str(counter).""" while True: name = prefix if counter > 0: name += '_' + str(counter) if self.try_claim_module_name(name, owner): return name counter += 1 def find_root_pkg(self): """Read name of [package] in ./Cargo.toml.""" if not os.path.exists('./Cargo.toml'): return with open('./Cargo.toml', 'r') as inf: pkg_section = re.compile(r'^ *\[package\]') name = re.compile('^ *name *= * "([^"]*)"') in_pkg = False for line in inf: if in_pkg: if name.match(line): self.root_pkg = name.match(line).group(1) break else: in_pkg = pkg_section.match(line) is not None def update_variant_args(self, variant_num): if 'variants' in self.args: # Resolve - and _ for Namespace usage variant_data = {k.replace('-', '_') : v for k, v in self.args.variants[variant_num].items()} # Merge and overwrite variant args self.variant_args = argparse.Namespace(**{**vars(self.args), **variant_data}) # Default action is cargo clean, followed by build or user given actions. if self.variant_args.cargo: self.cargo = ['clean'] + self.variant_args.cargo else: default_target = '--target x86_64-unknown-linux-gnu' # Use the same target for both host and default device builds. # Same target is used as default in host x86_64 Android compilation. # Note: b/169872957, prebuilt cargo failed to build vsock # on x86_64-unknown-linux-musl systems. self.cargo = ['clean', 'build ' + default_target] if self.variant_args.tests: self.cargo.append('build --tests ' + default_target) def run_cargo(self): """Calls cargo -v and save its output to ./cargo{_variant_num}.out.""" if self.skip_cargo: return self num_variants = len(self.args.variants) if 'variants' in self.args else 1 for variant_num in range(num_variants): cargo_toml = './Cargo.toml' cargo_out = './cargo.out' if variant_num > 0: cargo_out = f'./cargo_{variant_num}.out' self.variant_num = variant_num self.update_variant_args(variant_num=variant_num) # Do not use Cargo.lock, because .bp rules are designed to # run with "latest" crates available on Android. cargo_lock = './Cargo.lock' cargo_lock_saved = './cargo.lock.saved' had_cargo_lock = os.path.exists(cargo_lock) if not os.access(cargo_toml, os.R_OK): print('ERROR: Cannot find or read', cargo_toml) return self if not self.dry_run: if os.path.exists(cargo_out): os.remove(cargo_out) if not self.variant_args.use_cargo_lock and had_cargo_lock: # save it os.rename(cargo_lock, cargo_lock_saved) cmd_tail_target = ' --target-dir ' + TARGET_TMP cmd_tail_redir = ' >> ' + cargo_out + ' 2>&1' # set up search PATH for cargo to find the correct rustc saved_path = os.environ['PATH'] os.environ['PATH'] = os.path.dirname(self.cargo_path) + ':' + saved_path # Add [workspace] to Cargo.toml if it is not there. added_workspace = False if self.variant_args.add_workspace: with open(cargo_toml, 'r') as in_file: cargo_toml_lines = in_file.readlines() found_workspace = '[workspace]\n' in cargo_toml_lines if found_workspace: print('### WARNING: found [workspace] in Cargo.toml') else: with open(cargo_toml, 'a') as out_file: out_file.write('\n\n[workspace]\n') added_workspace = True if self.variant_args.verbose: print('### INFO: added [workspace] to Cargo.toml') for c in self.cargo: features = '' if c != 'clean': if self.variant_args.features is not None: features = ' --no-default-features' if self.variant_args.features: features += ' --features ' + self.variant_args.features cmd_v_flag = ' -vv ' if self.variant_args.vv else ' -v ' cmd = self.cargo_path + cmd_v_flag cmd += c + features + cmd_tail_target + cmd_tail_redir if self.variant_args.rustflags and c != 'clean': cmd = 'RUSTFLAGS="' + self.variant_args.rustflags + '" ' + cmd self.run_cmd(cmd, cargo_out) if self.variant_args.tests: cmd = self.cargo_path + ' test' + features + cmd_tail_target + ' -- --list' + cmd_tail_redir self.run_cmd(cmd, cargo_out) if added_workspace: # restore original Cargo.toml with open(cargo_toml, 'w') as out_file: out_file.writelines(cargo_toml_lines) if self.variant_args.verbose: print('### INFO: restored original Cargo.toml') os.environ['PATH'] = saved_path if not self.dry_run: if not had_cargo_lock: # restore to no Cargo.lock state if os.path.exists(cargo_lock): os.remove(cargo_lock) elif not self.variant_args.use_cargo_lock: # restore saved Cargo.lock os.rename(cargo_lock_saved, cargo_lock) return self def run_cmd(self, cmd, cargo_out): if self.dry_run: print('Dry-run skip:', cmd) else: if self.variant_args.verbose: print('Running:', cmd) with open(cargo_out, 'a+') as out_file: out_file.write('### Running: ' + cmd + '\n') ret = os.system(cmd) if ret != 0: print('*** There was an error while running cargo. ' + 'See the cargo.out file for details.') def dump_pkg_obj2cc(self): """Dump debug info of the pkg_obj2cc map.""" if not self.variant_args.debug: return self.init_bp_file('Android.bp') with open('Android.bp', 'a') as outf: sorted_pkgs = sorted(self.pkg_obj2cc.keys()) for pkg in sorted_pkgs: if not self.pkg_obj2cc[pkg]: continue outf.write('\n// obj => src for %s\n' % pkg) obj2cc = self.pkg_obj2cc[pkg] for obj in sorted(obj2cc.keys()): outf.write('// ' + short_out_name(pkg, obj) + ' => ' + short_out_name(pkg, obj2cc[obj].src) + '\n') def apply_patch(self): """Apply local patch file if it is given.""" if self.args.patch: if self.dry_run: print('Dry-run skip patch file:', self.args.patch) else: if not os.path.exists(self.args.patch): self.append_to_bp('ERROR cannot find patch file: ' + self.args.patch) return self if self.args.verbose: print('### INFO: applying local patch file:', self.args.patch) subprocess.run(['patch', '-s', '--no-backup-if-mismatch', './Android.bp', self.args.patch], check=True) return self def gen_bp(self): """Parse cargo.out and generate Android.bp files.""" if self.dry_run: print('Dry-run skip: read', CARGO_OUT, 'write Android.bp') elif os.path.exists(CARGO_OUT): self.find_root_pkg() if self.args.copy_out: self.copy_out_files() elif self.find_out_files() and self.has_used_out_dir(): print('WARNING: ' + self.root_pkg + ' has cargo output files; ' + 'please rerun with the --copy-out flag.') num_variants = len(self.args.variants) if 'variants' in self.args else 1 for variant_num in range(num_variants): cargo_out_path = CARGO_OUT if variant_num > 0: cargo_out_path = f'./cargo_{variant_num}.out' self.variant_num = variant_num self.update_variant_args(variant_num=variant_num) with open(cargo_out_path, 'r') as cargo_out: self.parse(cargo_out, 'Android.bp') self.crates.sort(key=get_module_name) for obj in self.cc_objects: obj.dump() self.dump_pkg_obj2cc() for crate in self.crates: self.update_variant_args(variant_num=crate.variant_num) crate.dump() dumped_libs = set() for lib in self.ar_objects: if lib.pkg == self.root_pkg: lib_name = file_base_name(lib.lib) if lib_name not in dumped_libs: dumped_libs.add(lib_name) lib.dump() if self.args.add_toplevel_block: with open(self.args.add_toplevel_block, 'r') as f: self.append_to_bp('\n' + f.read() + '\n') if self.errors: self.append_to_bp('\n' + ERRORS_LINE + '\n' + self.errors) if self.test_errors: self.append_to_bp('\n// Errors when listing tests:\n' + self.test_errors) return self def add_ar_object(self, obj): self.ar_objects.append(obj) def add_cc_object(self, obj): self.cc_objects.append(obj) def add_crate(self, crate): """Merge crate with someone in crates, or append to it. Return crates.""" if crate.skip_crate(): if self.args.debug: # include debug info of all crates self.crates.append(crate) else: for c in self.crates: if c.merge(crate, 'Android.bp'): return # If not merged, decide module type and name now. crate.decide_module_type() self.crates.append(crate) def find_warning_owners(self): """For each warning file, find its owner crate.""" missing_owner = False for f in self.warning_files: cargo_dir = '' # find lowest crate, with longest path owner = None # owner crate of this warning for c in self.crates: if (f.startswith(c.cargo_dir + '/') and len(cargo_dir) < len(c.cargo_dir)): cargo_dir = c.cargo_dir owner = c if owner: owner.has_warning = True else: missing_owner = True if missing_owner and os.path.exists('Cargo.toml'): # owner is the root cargo, with empty cargo_dir for c in self.crates: if not c.cargo_dir: c.has_warning = True def rustc_command(self, n, rustc_line, line, outf_name): """Process a rustc command line from cargo -vv output.""" # cargo build -vv output can have multiple lines for a rustc command # due to '\n' in strings for environment variables. # strip removes leading spaces and '\n' at the end new_rustc = (rustc_line.strip() + line) if rustc_line else line # Use an heuristic to detect the completions of a multi-line command. # This might fail for some very rare case, but easy to fix manually. if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0: return new_rustc if RUSTC_VV_CMD_ARGS.match(new_rustc): args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(2) self.add_crate(Crate(self, outf_name).parse(n, args)) else: self.assert_empty_vv_line(new_rustc) return '' def cc_ar_command(self, n, groups, outf_name): pkg = groups.group(1) line = groups.group(3) if groups.group(2) == 'cc': self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line)) else: self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line)) def append_to_bp(self, line): self.init_bp_file('Android.bp') with open('Android.bp', 'a') as outf: outf.write(line) def assert_empty_vv_line(self, line): if line: # report error if line is not empty self.append_to_bp('ERROR -vv line: ' + line) return '' def add_empty_test(self, name): if name.startswith('unittests'): self.empty_unittests = True else: self.empty_tests.add(name) def should_ignore_test(self, src): # cargo test outputs the source file for integration tests but "unittests" # for unit tests. To figure out to which crate this corresponds, we check # if the current source file is the main source of a non-test crate, e.g., # a library or a binary. return (src in self.variant_args.test_blocklist or src in self.empty_tests or (self.empty_unittests and src in [c.main_src for c in self.crates if c.crate_types != ['test']])) def parse(self, inf, outf_name): """Parse rustc, test, and warning messages in inf, return a list of Crates.""" n = 0 # line number # We read the file in two passes, where the first simply checks for empty tests. # Otherwise we would add and merge tests before seeing they're empty. cur_test_name = None for line in inf: if CARGO_TEST_LIST_START_PAT.match(line): cur_test_name = CARGO_TEST_LIST_START_PAT.match(line).group(1) elif cur_test_name and CARGO_TEST_LIST_END_PAT.match(line): match = CARGO_TEST_LIST_END_PAT.match(line) if int(match.group(1)) + int(match.group(2)) == 0: self.add_empty_test(cur_test_name) cur_test_name = None inf.seek(0) prev_warning = False # true if the previous line was warning: ... rustc_line = '' # previous line(s) matching RUSTC_VV_PAT in_tests = False for line in inf: n += 1 if line.startswith('warning: '): prev_warning = True rustc_line = self.assert_empty_vv_line(rustc_line) continue new_rustc = '' if RUSTC_PAT.match(line): args_line = RUSTC_PAT.match(line).group(2) self.add_crate(Crate(self, outf_name).parse(n, args_line)) self.assert_empty_vv_line(rustc_line) elif rustc_line or RUSTC_VV_PAT.match(line): new_rustc = self.rustc_command(n, rustc_line, line, outf_name) elif CC_AR_VV_PAT.match(line): self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name) elif prev_warning and WARNING_FILE_PAT.match(line): self.assert_empty_vv_line(rustc_line) fpath = WARNING_FILE_PAT.match(line).group(1) if fpath[0] != '/': # ignore absolute path self.warning_files.add(fpath) elif line.startswith('error: ') or line.startswith('error[E'): if not self.args.ignore_cargo_errors: if in_tests: self.test_errors += '// ' + line else: self.errors += line elif CARGO2ANDROID_RUNNING_PAT.match(line): in_tests = "cargo test" in line and "--list" in line prev_warning = False rustc_line = new_rustc self.find_warning_owners() def get_parser() -> argparse.ArgumentParser: """Parse main arguments.""" parser = argparse.ArgumentParser('cargo2android') parser.add_argument( '--add_workspace', action='store_true', default=False, help=('append [workspace] to Cargo.toml before calling cargo,' + ' to treat current directory as root of package source;' + ' otherwise the relative source file path in generated' + ' .bp file will be from the parent directory.')) parser.add_argument( '--cargo', action='append', metavar='args_string', help=('extra cargo build -v args in a string, ' + 'each --cargo flag calls cargo build -v once')) parser.add_argument( '--cargo_bin', type=str, help='use cargo in the cargo_bin directory instead of the prebuilt one') parser.add_argument( '--copy-out', action='store_true', default=False, help=('only for root directory, ' + 'copy build.rs output to ./out/* and add a genrule to copy ' + './out/* to genrule output; for crates with code pattern: ' + 'include!(concat!(env!("OUT_DIR"), "/.rs"))')) parser.add_argument( '--debug', action='store_true', default=False, help='dump debug info into Android.bp') parser.add_argument( '--dependencies', action='store_true', default=False, help='Deprecated. Has no effect.') parser.add_argument( '--device', action='store_true', default=False, help='run cargo also for a default device target') parser.add_argument( '--features', type=str, help=('pass features to cargo build, ' + 'empty string means no default features')) parser.add_argument( '--global_defaults', type=str, help='add a defaults name to every module') parser.add_argument( '--host-first-multilib', action='store_true', default=False, help=('add a compile_multilib:"first" property ' + 'to Android.bp host modules.')) parser.add_argument( '--ignore-cargo-errors', action='store_true', default=False, help='do not append cargo/rustc error messages to Android.bp') parser.add_argument( '--no-host', action='store_true', default=False, help='do not run cargo for the host; only for the device target') parser.add_argument( '--no-presubmit', action='store_true', default=False, help='set unit_test to false for test targets, to avoid host tests running in presubmit') parser.add_argument( '--no-subdir', action='store_true', default=False, help='do not output anything for sub-directories') parser.add_argument( '--onefile', action='store_true', default=False, help=('output all into one ./Android.bp, default will generate ' + 'one Android.bp per Cargo.toml in subdirectories')) parser.add_argument( '--patch', type=str, help='apply the given patch file to generated ./Android.bp') parser.add_argument( '--run', action='store_true', default=False, help='run it, default is dry-run') parser.add_argument('--rustflags', type=str, help='passing flags to rustc') parser.add_argument( '--skipcargo', action='store_true', default=False, help='skip cargo command, parse cargo.out, and generate Android.bp') parser.add_argument( '--tests', action='store_true', default=False, help='run cargo build --tests after normal build') parser.add_argument( '--use-cargo-lock', action='store_true', default=False, help=('run cargo build with existing Cargo.lock ' + '(used when some latest dependent crates failed)')) parser.add_argument( '--exported_c_header_dir', nargs='*', help='Directories with headers to export for C usage' ) parser.add_argument( '--min-sdk-version', type=str, help='Minimum SDK version') parser.add_argument( '--apex-available', nargs='*', help='Mark the main library as apex_available with the given apexes.') parser.add_argument( '--native-bridge-supported', action='store_true', default=False, help='Mark the main library as native_bridge_supported.') parser.add_argument( '--product-available', action='store_true', default=True, help='Mark the main library as product_available.') parser.add_argument( '--recovery-available', action='store_true', default=False, help='Mark the main library as recovery_available.') parser.add_argument( '--vendor-available', action='store_true', default=True, help='Mark the main library as vendor_available.') parser.add_argument( '--vendor-ramdisk-available', action='store_true', default=False, help='Mark the main library as vendor_ramdisk_available.') parser.add_argument( '--ramdisk-available', action='store_true', default=False, help='Mark the main library as ramdisk_available.') parser.add_argument( '--force-rlib', action='store_true', default=False, help='Make the main library an rlib.') parser.add_argument( '--whole-static-libs', nargs='*', default=[], help='Make the given libraries (without lib prefixes) whole_static_libs.') parser.add_argument( '--no-pkg-vers', action='store_true', default=False, help='Do not attempt to determine the package version automatically.') parser.add_argument( '--test-data', nargs='*', default=[], help=('Add the given file to the given test\'s data property. ' + 'Usage: test-path=data-path')) parser.add_argument( '--dependency-blocklist', nargs='*', default=[], help='Do not emit the given dependencies (without lib prefixes).') parser.add_argument( '--lib-blocklist', nargs='*', default=[], help='Do not emit the given C libraries as dependencies (without lib prefixes).') parser.add_argument( '--test-blocklist', nargs='*', default=[], help=('Do not emit the given tests. ' + 'Pass the path to the test file to exclude.')) parser.add_argument( '--cfg-blocklist', nargs='*', default=[], help='Do not emit the given cfg.') parser.add_argument( '--add-toplevel-block', type=str, help=('Add the contents of the given file to the top level of the Android.bp. ' + 'The filename should start with cargo2android to work with the updater.')) parser.add_argument( '--add-module-block', type=str, help=('Add the contents of the given file to the main module. '+ 'The filename should start with cargo2android to work with the updater.')) parser.add_argument( '--no-std', action='store_true', default=False, help='Don\'t link against std.') parser.add_argument( '--alloc', action='store_true', default=False, help='Link against alloc. Only valid if --no-std is also passed.') parser.add_argument( '--dependency-suffix', type=str, help='Suffix to add to name of dependencies') parser.add_argument( '--verbose', action='store_true', default=False, help='echo executed commands') parser.add_argument( '--vv', action='store_true', default=False, help='run cargo with -vv instead of default -v') parser.add_argument( '--dump-config-and-exit', type=str, help=('Dump command-line arguments (minus this flag) to a config file and exit. ' + 'This is intended to help migrate from command line options to config files.')) parser.add_argument( '--name-suffix', type=str, default='', help=('Add this suffix to the module name.')) parser.add_argument( '--dep-suffixes', default={}, help=('Add these suffixes to the specified dependencies')) parser.add_argument( '--config', type=str, help=('Load command-line options from the given config file. ' + 'Options in this file will override those passed on the command line.')) return parser def parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace: """Parses command-line options.""" args = parser.parse_args() # Use the values specified in a config file if one was found. if args.config: with open(args.config, 'r') as f: config = json.load(f) args_dict = vars(args) for arg in config: args_dict[arg.replace('-', '_')] = config[arg] return args def dump_config(parser: argparse.ArgumentParser, args): """Writes the non-default command-line options to the specified file.""" args_dict = vars(args) # Filter out the arguments that have their default value. # Also filter certain "temporary" arguments. non_default_args = {} for arg in args_dict: if (args_dict[arg] != parser.get_default(arg) and arg != 'dump_config_and_exit' and arg != 'config' and arg != 'cargo_bin'): non_default_args[arg.replace('_', '-')] = args_dict[arg] # Write to the specified file. with open(args.dump_config_and_exit, 'w') as f: json.dump(non_default_args, f, indent=2, sort_keys=True) def main(): parser = get_parser() args = parse_args(parser) if not args.run: # default is dry-run print(DRY_RUN_NOTE) if args.dump_config_and_exit: dump_config(parser, args) else: Runner(args).run_cargo().gen_bp().apply_patch() if __name__ == '__main__': main()