#!/usr/bin/env python3 # Copyright (C) 2023 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Call cargo -v, parse its output, and generate a Trusty build system module. Usage: Run this script in a crate workspace root directory. The Cargo.toml file should work at least for the host platform. Without other flags, "cargo2rulesmk.py --run" calls cargo clean, calls cargo build -v, and generates makefile rules. The cargo build only generates crates for the host without test crates. If there are rustc warning messages, this script will add a warning comment to the owner crate module in rules.mk. """ import argparse import glob import json import os import os.path import platform import re import shutil import subprocess import sys from typing import List assert "/development/scripts" in os.path.dirname(__file__) TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) # Some Rust packages include extra unwanted crates. # This set contains all such excluded crate names. EXCLUDED_CRATES = {"protobuf_bin_gen_rust_do_not_use"} CUSTOM_MODULE_CRATES = { # This map tracks Rust crates that have special modules that # were not generated automatically by this script. Examples # include compiler builtins and other foundational libraries. # It also tracks crates tht are not under external/rust/crates. "compiler_builtins": "trusty/user/base/lib/libcompiler_builtins-rust", "core": "trusty/user/base/lib/libcore-rust", } RENAME_STEM_MAP = { # This map includes all changes to the default rust module stem names, # which is used for output files when different from the module name. "protoc_gen_rust": "protoc-gen-rust", } # Header added to all generated rules.mk files. RULES_MK_HEADER = ( "# This file is generated by cargo2rulesmk.py {args}.\n" + "# Do not modify this file as changes will be overridden on upgrade.\n\n" ) CARGO_OUT = "cargo.out" # Name of file to keep cargo build -v output. # This should be kept in sync with tools/external_updater/crates_updater.py. ERRORS_LINE = "Errors in " + CARGO_OUT + ":" TARGET_TMP = "target.tmp" # Name of temporary output directory. # Message to be displayed when this script is called without the --run flag. DRY_RUN_NOTE = ( "Dry-run: This script uses ./" + TARGET_TMP + " for output directory,\n" + "runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n" + "and writes to rules.mk in the current and subdirectories.\n\n" + "To do do all of the above, use the --run flag.\n" + "See --help for other flags, and more usage notes in this script.\n" ) # Cargo -v output of a call to rustc. RUSTC_PAT = re.compile("^ +Running `(.*\/)?rustc (.*)`$") # Cargo -vv output of a call to rustc could be split into multiple lines. # Assume that the first line will contain some CARGO_* env definition. RUSTC_VV_PAT = re.compile("^ +Running `.*CARGO_.*=.*$") # The combined -vv output rustc command line pattern. RUSTC_VV_CMD_ARGS = re.compile("^ *Running `.*CARGO_.*=.* (.*\/)?rustc (.*)`$") # Cargo -vv output of a "cc" or "ar" command; all in one line. CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$') # Some package, such as ring-0.13.5, has pattern '... running "cc"'. # Rustc output of file location path pattern for a warning message. WARNING_FILE_PAT = re.compile("^ *--> ([^:]*):[0-9]+") # cargo test --list output of the start of running a binary. CARGO_TEST_LIST_START_PAT = re.compile(r"^\s*Running (.*) \(.*\)$") # cargo test --list output of the end of running a binary. CARGO_TEST_LIST_END_PAT = re.compile(r"^(\d+) tests?, (\d+) benchmarks$") CARGO2ANDROID_RUNNING_PAT = re.compile("^### Running: .*$") # Rust package name with suffix -d1.d2.d3(+.*)?. VERSION_SUFFIX_PAT = re.compile( r"^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:-(alpha|beta)\.[0-9]+)?(?:\+.*)?$" ) # Crate types corresponding to a C ABI library C_LIBRARY_CRATE_TYPES = ["staticlib", "cdylib"] # Crate types corresponding to a Rust ABI library RUST_LIBRARY_CRATE_TYPES = ["lib", "rlib", "dylib", "proc-macro"] # Crate types corresponding to a library LIBRARY_CRATE_TYPES = C_LIBRARY_CRATE_TYPES + RUST_LIBRARY_CRATE_TYPES def altered_stem(name): return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name def is_build_crate_name(name): # We added special prefix to build script crate names. return name.startswith("build_script_") def is_dependent_file_path(path): # Absolute or dependent '.../' paths are not main files of this crate. return path.startswith("/") or path.startswith(".../") def get_module_name(crate): # to sort crates in a list return crate.module_name def pkg2crate_name(s): return s.replace("-", "_").replace(".", "_") def file_base_name(path): return os.path.splitext(os.path.basename(path))[0] def test_base_name(path): return pkg2crate_name(file_base_name(path)) def unquote(s): # remove quotes around str if s and len(s) > 1 and s[0] == s[-1] and s[0] in ('"', "'"): return s[1:-1] return s def remove_version_suffix(s): # remove -d1.d2.d3 suffix if match := VERSION_SUFFIX_PAT.match(s): return match.group(1) return s def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/* return re.sub("^/.*/" + pkg + "-[0-9a-f]*/out/", ".../out/", s) class Crate(object): """Information of a Rust crate to collect/emit for a rules.mk module.""" def __init__(self, runner, outf_name): # Remembered global runner and its members. self.runner = runner self.debug = runner.args.debug self.cargo_dir = "" # directory of my Cargo.toml self.outf_name = outf_name # path to rules.mk self.outf = None # open file handle of outf_name during dump* self.has_warning = False # Trusty module properties derived from rustc parameters. self.module_name = "" self.defaults = "" # rust_defaults used by rust_test* modules self.default_srcs = False # use 'srcs' defined in self.defaults self.root_pkg = "" # parent package name of a sub/test packge, from -L self.srcs = [] # main_src or merged multiple source files self.stem = "" # real base name of output file # Kept parsed status self.errors = "" # all errors found during parsing self.line_num = 1 # runner told input source line number self.line = "" # original rustc command line parameters # Parameters collected from rustc command line. self.crate_name = "" # follows --crate-name self.main_src = "" # follows crate_name parameter, shortened self.crate_types = [] # follows --crate-type self.cfgs = [] # follows --cfg, without feature= prefix self.features = [] # follows --cfg, name in 'feature="..."' self.codegens = [] # follows -C, some ignored self.static_libs = [] # e.g. -l static=host_cpuid self.shared_libs = [] # e.g. -l dylib=wayland-client, -l z self.cap_lints = "" # follows --cap-lints self.emit_list = "" # e.g., --emit=dep-info,metadata,link self.edition = "2015" # rustc default, e.g., --edition=2018 self.target = "" # follows --target self.cargo_env_compat = True # Parameters collected from cargo metadata output self.dependencies = [] # crate dependencies output by `cargo metadata` self.feature_dependencies: dict[str, List[str]] = {} # maps features to # optional dependencies def write(self, s): """convenient way to output one line at a time with EOL.""" assert self.outf self.outf.write(s + "\n") def find_cargo_dir(self): """Deepest directory with Cargo.toml and contains the main_src.""" if not is_dependent_file_path(self.main_src): dir_name = os.path.dirname(self.main_src) while dir_name: if os.path.exists(dir_name + "/Cargo.toml"): self.cargo_dir = dir_name return dir_name = os.path.dirname(dir_name) def add_codegens_flag(self, flag): """Ignore options not used by Trusty build system""" # 'prefer-dynamic' may be set by library.mk # 'embed-bitcode' is ignored; we might control LTO with other flags # 'codegen-units' is set globally in engine.mk # 'relocation-model' and 'target-feature=+reserve-x18' may be set by # common_flags.mk if not ( flag.startswith("codegen-units=") or flag.startswith("debuginfo=") or flag.startswith("embed-bitcode=") or flag.startswith("extra-filename=") or flag.startswith("incremental=") or flag.startswith("metadata=") or flag.startswith("relocation-model=") or flag == "prefer-dynamic" or flag == "target-feature=+reserve-x18" ): self.codegens.append(flag) def get_dependencies(self): """Use output from cargo metadata to determine crate dependencies""" cargo_metadata = subprocess.run( [ self.runner.cargo_path, "metadata", "--no-deps", "--format-version", "1", ], cwd=os.path.abspath(self.cargo_dir), stdout=subprocess.PIPE, check=False, ) if cargo_metadata.returncode: self.errors += ( "ERROR: unable to get cargo metadata to determine " f"dependencies; return code {cargo_metadata.returncode}\n" ) else: metadata_json = json.loads(cargo_metadata.stdout) for package in metadata_json["packages"]: # package names containing '-' are changed to '_' in crate_name if package["name"].replace("-", "_") == self.crate_name: self.dependencies = package["dependencies"] for feat, props in package["features"].items(): feat_deps = [ d[4:] for d in props if d.startswith("dep:") ] if feat_deps and feat in self.feature_dependencies: self.feature_dependencies[feat].extend(feat_deps) else: self.feature_dependencies[feat] = feat_deps break else: # package name not found in metadata if is_build_crate_name(self.crate_name): print( "### WARNING: unable to determine dependencies for " + f"{self.crate_name} from cargo metadata" ) def parse(self, line_num, line): """Find important rustc arguments to convert to makefile rules.""" self.line_num = line_num self.line = line args = [unquote(l) for l in line.split()] i = 0 # Loop through every argument of rustc. while i < len(args): arg = args[i] if arg == "--crate-name": i += 1 self.crate_name = args[i] elif arg == "--crate-type": i += 1 # cargo calls rustc with multiple --crate-type flags. # rustc can accept: # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro] self.crate_types.append(args[i]) elif arg == "--test": self.crate_types.append("test") elif arg == "--target": i += 1 self.target = args[i] elif arg == "--cfg": i += 1 if args[i].startswith("feature="): self.features.append( unquote(args[i].replace("feature=", "")) ) else: self.cfgs.append(args[i]) elif arg == "--extern": i += 1 pass # ignored; get all dependencies from cargo metadata elif arg == "-C": # codegen options i += 1 self.add_codegens_flag(args[i]) elif arg.startswith("-C"): # cargo has been passing "-C " flag to rustc, # but newer cargo could pass '-Cembed-bitcode=no' to rustc. self.add_codegens_flag(arg[2:]) elif arg == "--cap-lints": i += 1 self.cap_lints = args[i] elif arg == "-L": i += 1 if args[i].startswith("dependency=") and args[i].endswith( "/deps" ): if "/" + TARGET_TMP + "/" in args[i]: self.root_pkg = re.sub( "^.*/", "", re.sub("/" + TARGET_TMP + "/.*/deps$", "", args[i]), ) else: self.root_pkg = re.sub( "^.*/", "", re.sub("/[^/]+/[^/]+/deps$", "", args[i]), ) self.root_pkg = remove_version_suffix(self.root_pkg) elif arg == "-l": i += 1 if args[i].startswith("static="): self.static_libs.append(re.sub("static=", "", args[i])) elif args[i].startswith("dylib="): self.shared_libs.append(re.sub("dylib=", "", args[i])) else: self.shared_libs.append(args[i]) elif arg in ("--out-dir", "--color"): # ignored i += 1 elif arg.startswith("--error-format=") or arg.startswith("--json="): pass # ignored elif arg.startswith("--emit="): self.emit_list = arg.replace("--emit=", "") elif arg.startswith("--edition="): self.edition = arg.replace("--edition=", "") elif arg.startswith("-Aclippy") or arg.startswith("-Wclippy"): pass # TODO: emit these flags in rules.mk elif arg.startswith("-W"): pass # ignored elif arg.startswith("-Z"): pass # ignore unstable flags elif arg.startswith("-D"): pass # TODO: emit these flags in rules.mk elif not arg.startswith("-"): # shorten imported crate main source paths like $HOME/.cargo/ # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/ # lib.rs self.main_src = re.sub( r"^/[^ ]*/registry/src/", ".../", args[i] ) self.main_src = re.sub( r"^\.\.\./github.com-[0-9a-f]*/", ".../", self.main_src ) self.find_cargo_dir() if self.cargo_dir: # for a subdirectory if ( self.runner.args.no_subdir ): # all .mk content to /dev/null self.outf_name = "/dev/null" elif not self.runner.args.onefile: # Write to rules.mk in the subdirectory with Cargo.toml. self.outf_name = self.cargo_dir + "/rules.mk" self.main_src = self.main_src[len(self.cargo_dir) + 1 :] else: self.errors += "ERROR: unknown " + arg + "\n" i += 1 if not self.crate_name: self.errors += "ERROR: missing --crate-name\n" if not self.main_src: self.errors += "ERROR: missing main source file\n" else: self.srcs.append(self.main_src) if not self.crate_types: # Treat "--cfg test" as "--test" if "test" in self.cfgs: self.crate_types.append("test") else: self.errors += "ERROR: missing --crate-type or --test\n" elif len(self.crate_types) > 1: if "test" in self.crate_types: self.errors += ( "ERROR: cannot handle both --crate-type and --test\n" ) if "lib" in self.crate_types and "rlib" in self.crate_types: self.errors += ( "ERROR: cannot generate both lib and rlib crate types\n" ) if not self.root_pkg: self.root_pkg = self.crate_name # get the package dependencies by running cargo metadata if not self.skip_crate(): self.get_dependencies() self.cfgs = sorted(set(self.cfgs)) self.features = sorted(set(self.features)) self.codegens = sorted(set(self.codegens)) self.static_libs = sorted(set(self.static_libs)) self.shared_libs = sorted(set(self.shared_libs)) self.crate_types = sorted(set(self.crate_types)) self.module_name = self.stem return self def dump_line(self): self.write("\n// Line " + str(self.line_num) + " " + self.line) def feature_list(self): """Return a string of main_src + "feature_list".""" pkg = self.main_src if pkg.startswith(".../"): # keep only the main package name pkg = re.sub("/.*", "", pkg[4:]) elif pkg.startswith("/"): # use relative path for a local package pkg = os.path.relpath(pkg) if not self.features: return pkg return pkg + ' "' + ",".join(self.features) + '"' def dump_skip_crate(self, kind): if self.debug: self.write("\n// IGNORED: " + kind + " " + self.main_src) return self def skip_crate(self): """Return crate_name or a message if this crate should be skipped.""" if ( is_build_crate_name(self.crate_name) or self.crate_name in EXCLUDED_CRATES ): return self.crate_name if is_dependent_file_path(self.main_src): return "dependent crate" return "" def dump(self): """Dump all error/debug/module code to the output rules.mk file.""" self.runner.init_rules_file(self.outf_name) with open(self.outf_name, "a", encoding="utf-8") as outf: self.outf = outf if self.errors: self.dump_line() self.write(self.errors) elif self.skip_crate(): self.dump_skip_crate(self.skip_crate()) else: if self.debug: self.dump_debug_info() self.dump_trusty_module() self.outf = None def dump_debug_info(self): """Dump parsed data, when cargo2rulesmk is called with --debug.""" def dump(name, value): self.write(f"//{name:>12} = {value}") def opt_dump(name, value): if value: dump(name, value) def dump_list(fmt, values): for v in values: self.write(fmt % v) self.dump_line() dump("module_name", self.module_name) dump("crate_name", self.crate_name) dump("crate_types", self.crate_types) dump("main_src", self.main_src) dump("has_warning", self.has_warning) opt_dump("target", self.target) opt_dump("edition", self.edition) opt_dump("emit_list", self.emit_list) opt_dump("cap_lints", self.cap_lints) dump_list("// cfg = %s", self.cfgs) dump_list("// cfg = 'feature \"%s\"'", self.features) # TODO(chh): escape quotes in self.features, but not in other dump_list dump_list("// codegen = %s", self.codegens) dump_list("// -l static = %s", self.static_libs) dump_list("// -l (dylib) = %s", self.shared_libs) def dump_trusty_module(self): """Dump one or more module definitions, depending on crate_types.""" if len(self.crate_types) > 1: if "test" in self.crate_types: self.write("\nERROR: multiple crate types cannot include test type") return if "lib" in self.crate_types: print(f"### WARNING: crate {self.crate_name} has multiple " f"crate types ({str(self.crate_types)}). Treating as 'lib'") self.crate_types = ["lib"] else: self.write("\nERROR: don't know how to handle crate types of " f"crate {self.crate_name}: {str(self.crate_types)}") return self.dump_single_type_trusty_module() def dump_srcs_list(self): """Dump the srcs list, for defaults or regular modules.""" if len(self.srcs) > 1: srcs = sorted(set(self.srcs)) # make a copy and dedup else: srcs = [self.main_src] self.write("MODULE_SRCS := \\") for src in srcs: self.write(f"\t$(LOCAL_DIR)/{src} \\") self.write("") # add rust file generated by build.rs to MODULE_SRCDEPS, if any # TODO(perlarsen): is there a need to support more than one output file? if srcdeps := [ f for f in self.runner.build_out_files if f.endswith(".rs") ]: assert len(srcdeps) == 1 outfile = srcdeps.pop() lines = [ f"OUT_FILE := $(call TOBUILDDIR,$(MODULE))/{outfile}", f"$(OUT_FILE): $(MODULE)/out/{outfile}", "\t@echo copying $< to $@", "\t@$(MKDIR)", "\tcp $< $@", "", "MODULE_RUST_ENV += OUT_DIR=$(dir $(OUT_FILE))", "", "MODULE_SRCDEPS := $(OUT_FILE)", ] self.write("\n".join(lines)) def dump_single_type_trusty_module(self): """Dump one simple Trusty module, which has only one crate_type.""" crate_type = self.crate_types[0] assert crate_type != "test" self.dump_one_trusty_module(crate_type) def dump_one_trusty_module(self, crate_type): """Dump one Trusty module definition.""" if crate_type in ["test", "bin"]: # TODO: support test crates print( f"### WARNING: ignoring {crate_type} crate: {self.crate_name}") return if self.codegens: # TODO: support crates that require codegen flags print( f"ERROR: {self.crate_name} uses unexpected codegen flags: " + str(self.codegens) ) return self.dump_core_properties() if not self.defaults: self.dump_edition_flags_libs() # NOTE: a crate may list the same dependency as required and optional library_deps = set() for dependency in self.dependencies: if dependency["kind"] in ["dev", "build"]: continue name = ( rename if (rename := dependency["rename"]) else dependency["name"] ) if dependency["target"]: print( f"### WARNING: ignoring target-specific dependency: {name}") continue path = CUSTOM_MODULE_CRATES.get( name, f"external/rust/crates/{name}" ) if dependency["optional"]: if not any( name in self.feature_dependencies.get(f, []) for f in self.features ): continue library_deps.add(path) if library_deps: self.write("MODULE_LIBRARY_DEPS := \\") for path in sorted(library_deps): self.write(f"\t{path} \\") self.write("") if crate_type == "test" and not self.default_srcs: raise NotImplementedError("Crates with test data are not supported") assert crate_type in LIBRARY_CRATE_TYPES self.write("include make/library.mk") def dump_edition_flags_libs(self): if self.edition: self.write(f"MODULE_RUST_EDITION := {self.edition}") if self.features or self.cfgs: self.write("MODULE_RUSTFLAGS += \\") for feature in self.features: self.write(f"\t--cfg 'feature=\"{feature}\"' \\") for cfg in self.cfgs: self.write(f"\t--cfg '{cfg}' \\") self.write("") if self.static_libs or self.shared_libs: print("### WARNING: Crates with depend on static or shared " "libraries are not supported") def main_src_basename_path(self): return re.sub("/", "_", re.sub(".rs$", "", self.main_src)) def test_module_name(self): """Return a unique name for a test module.""" # root_pkg+(_host|_device) + '_test_'+source_file_name suffix = self.main_src_basename_path() return self.root_pkg + "_test_" + suffix def dump_core_properties(self): """Dump the module header, name, stem, etc.""" self.write("LOCAL_DIR := $(GET_LOCAL_DIR)") self.write("MODULE := $(LOCAL_DIR)") self.write(f"MODULE_CRATE_NAME := {self.crate_name}") # Trusty's module system only supports bin, rlib, and proc-macro so map # lib->rlib if self.crate_types != ["lib"]: crate_types = set( "rlib" if ct == "lib" else ct for ct in self.crate_types ) self.write(f'MODULE_RUST_CRATE_TYPES := {" ".join(crate_types)}') if not self.default_srcs: self.dump_srcs_list() if hasattr(self.runner.args, "module_add_implicit_deps"): if hasattr(self.runner.args, "module_add_implicit_deps_reason"): self.write(self.runner.args.module_add_implicit_deps_reason) if self.runner.args.module_add_implicit_deps in [True, "yes"]: self.write("MODULE_ADD_IMPLICIT_DEPS := true") elif self.runner.args.module_add_implicit_deps in [False, "no"]: self.write("MODULE_ADD_IMPLICIT_DEPS := false") else: sys.exit( "ERROR: invalid value for module_add_implicit_deps: " + str(self.runner.args.module_add_implicit_deps) ) class Runner(object): """Main class to parse cargo -v output and print Trusty makefile modules.""" def __init__(self, args): self.mk_files = set() # Remember all Trusty module files. self.root_pkg = "" # name of package in ./Cargo.toml # Saved flags, modes, and data. self.args = args self.dry_run = not args.run self.skip_cargo = args.skipcargo self.cargo_path = "./cargo" # path to cargo, will be set later self.checked_out_files = False # to check only once self.build_out_files = [] # output files generated by build.rs self.crates: List[Crate] = [] self.warning_files = set() # Keep a unique mapping from (module name) to crate self.name_owners = {} # Save and dump all errors from cargo to rules.mk. self.errors = "" self.test_errors = "" self.setup_cargo_path() # Default action is cargo clean, followed by build or user given actions if args.cargo: self.cargo = ["clean"] + args.cargo else: default_target = "--target x86_64-unknown-linux-gnu" # Use the same target for both host and default device builds. # Same target is used as default in host x86_64 Android compilation. # Note: b/169872957, prebuilt cargo failed to build vsock # on x86_64-unknown-linux-musl systems. self.cargo = ["clean", "build " + default_target] if args.tests: self.cargo.append("build --tests " + default_target) self.empty_tests = set() self.empty_unittests = False def setup_cargo_path(self): """Find cargo in the --cargo_bin or prebuilt rust bin directory.""" if self.args.cargo_bin: self.cargo_path = os.path.join(self.args.cargo_bin, "cargo") if not os.path.isfile(self.cargo_path): sys.exit("ERROR: cannot find cargo in " + self.args.cargo_bin) print("INFO: using cargo in " + self.args.cargo_bin) return # TODO(perlarsen): try getting cargo from $RUST_BINDIR set in envsetup.sh # We have only tested this on Linux. if platform.system() != "Linux": sys.exit( "ERROR: this script has only been tested on Linux with cargo." ) # Assuming that this script is in development/scripts linux_dir = os.path.join( TOP_DIR, "prebuilts", "rust", "linux-x86" ) if not os.path.isdir(linux_dir): sys.exit("ERROR: cannot find directory " + linux_dir) rust_version = self.find_rust_version(linux_dir) if self.args.verbose: print(f"### INFO: using prebuilt rust version {rust_version}") cargo_bin = os.path.join(linux_dir, rust_version, "bin") self.cargo_path = os.path.join(cargo_bin, "cargo") if not os.path.isfile(self.cargo_path): sys.exit( "ERROR: cannot find cargo in " + cargo_bin + "; please try --cargo_bin= flag." ) return def find_rust_version(self, linux_dir): """find newest prebuilt rust version.""" # find the newest (largest) version number in linux_dir. rust_version = (0, 0, 0) # the prebuilt version to use version_pat = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)$") for dir_name in os.listdir(linux_dir): result = version_pat.match(dir_name) if not result: continue version = ( int(result.group(1)), int(result.group(2)), int(result.group(3)), ) if version > rust_version: rust_version = version return ".".join(str(ver) for ver in rust_version) def find_out_files(self): # list1 has build.rs output for normal crates list1 = glob.glob( TARGET_TMP + "/*/*/build/" + self.root_pkg + "-*/out/*" ) # list2 has build.rs output for proc-macro crates list2 = glob.glob(TARGET_TMP + "/*/build/" + self.root_pkg + "-*/out/*") return list1 + list2 def copy_out_files(self): """Copy build.rs output files to ./out and set up build_out_files.""" if self.checked_out_files: return self.checked_out_files = True cargo_out_files = self.find_out_files() out_files = set() if cargo_out_files: os.makedirs("out", exist_ok=True) for path in cargo_out_files: file_name = path.split("/")[-1] out_files.add(file_name) shutil.copy(path, "out/" + file_name) self.build_out_files = sorted(out_files) def has_used_out_dir(self): """Returns true if env!("OUT_DIR") is found.""" return 0 == os.system( "grep -rl --exclude build.rs --include \\*.rs" + " 'env!(\"OUT_DIR\")' * > /dev/null" ) def init_rules_file(self, name): # name could be rules.mk or sub_dir_path/rules.mk if name not in self.mk_files: self.mk_files.add(name) with open(name, "w", encoding="utf-8") as outf: print_args = sys.argv[1:].copy() if "--cargo_bin" in print_args: index = print_args.index("--cargo_bin") del print_args[index : index + 2] outf.write(RULES_MK_HEADER.format(args=" ".join(print_args))) def find_root_pkg(self): """Read name of [package] in ./Cargo.toml.""" if not os.path.exists("./Cargo.toml"): return with open("./Cargo.toml", "r", encoding="utf-8") as inf: pkg_section = re.compile(r"^ *\[package\]") name = re.compile('^ *name *= * "([^"]*)"') in_pkg = False for line in inf: if in_pkg: if match := name.match(line): self.root_pkg = match.group(1) break else: in_pkg = pkg_section.match(line) is not None def run_cargo(self): """Calls cargo -v and save its output to ./cargo.out.""" if self.skip_cargo: return self cargo_toml = "./Cargo.toml" cargo_out = "./cargo.out" # Do not use Cargo.lock, because Trusty makefile rules are designed # to run with the latest available vendored crates in Trusty. cargo_lock = "./Cargo.lock" cargo_lock_saved = "./cargo.lock.saved" had_cargo_lock = os.path.exists(cargo_lock) if not os.access(cargo_toml, os.R_OK): print("ERROR: Cannot find or read", cargo_toml) return self if not self.dry_run: if os.path.exists(cargo_out): os.remove(cargo_out) if not self.args.use_cargo_lock and had_cargo_lock: # save it os.rename(cargo_lock, cargo_lock_saved) cmd_tail_target = " --target-dir " + TARGET_TMP cmd_tail_redir = " >> " + cargo_out + " 2>&1" # set up search PATH for cargo to find the correct rustc saved_path = os.environ["PATH"] os.environ["PATH"] = os.path.dirname(self.cargo_path) + ":" + saved_path # Add [workspace] to Cargo.toml if it is not there. added_workspace = False cargo_toml_lines = None if self.args.add_workspace: with open(cargo_toml, "r", encoding="utf-8") as in_file: cargo_toml_lines = in_file.readlines() found_workspace = "[workspace]\n" in cargo_toml_lines if found_workspace: print("### WARNING: found [workspace] in Cargo.toml") else: with open(cargo_toml, "a", encoding="utf-8") as out_file: out_file.write("\n\n[workspace]\n") added_workspace = True if self.args.verbose: print("### INFO: added [workspace] to Cargo.toml") features = "" for c in self.cargo: features = "" if c != "clean": if self.args.features is not None: features = " --no-default-features" if self.args.features: features += " --features " + self.args.features cmd_v_flag = " -vv " if self.args.vv else " -v " cmd = self.cargo_path + cmd_v_flag cmd += c + features + cmd_tail_target + cmd_tail_redir if c != "clean": rustflags = self.args.rustflags if self.args.rustflags else "" # linting issues shouldn't prevent us from generating rules.mk rustflags = f'RUSTFLAGS="{rustflags} --cap-lints allow" ' cmd = rustflags + cmd self.run_cmd(cmd, cargo_out) if self.args.tests: cmd = ( self.cargo_path + " test" + features + cmd_tail_target + " -- --list" + cmd_tail_redir ) self.run_cmd(cmd, cargo_out) if added_workspace: # restore original Cargo.toml with open(cargo_toml, "w", encoding="utf-8") as out_file: assert cargo_toml_lines out_file.writelines(cargo_toml_lines) if self.args.verbose: print("### INFO: restored original Cargo.toml") os.environ["PATH"] = saved_path if not self.dry_run: if not had_cargo_lock: # restore to no Cargo.lock state if os.path.exists(cargo_lock): os.remove(cargo_lock) elif not self.args.use_cargo_lock: # restore saved Cargo.lock os.rename(cargo_lock_saved, cargo_lock) return self def run_cmd(self, cmd, cargo_out): if self.dry_run: print("Dry-run skip:", cmd) else: if self.args.verbose: print("Running:", cmd) with open(cargo_out, "a+", encoding="utf-8") as out_file: out_file.write("### Running: " + cmd + "\n") ret = os.system(cmd) if ret != 0: print( "*** There was an error while running cargo. " + f"See the {cargo_out} file for details." ) def apply_patch(self): """Apply local patch file if it is given.""" if self.args.patch: if self.dry_run: print("Dry-run skip patch file:", self.args.patch) else: if not os.path.exists(self.args.patch): self.append_to_rules( "ERROR cannot find patch file: " + self.args.patch ) return self if self.args.verbose: print( "### INFO: applying local patch file:", self.args.patch ) subprocess.run( [ "patch", "-s", "--no-backup-if-mismatch", "./rules.mk", self.args.patch, ], check=True, ) return self def gen_rules(self): """Parse cargo.out and generate Trusty makefile rules""" if self.dry_run: print("Dry-run skip: read", CARGO_OUT, "write rules.mk") elif os.path.exists(CARGO_OUT): self.find_root_pkg() if self.args.copy_out: self.copy_out_files() elif self.find_out_files() and self.has_used_out_dir(): print( "WARNING: " + self.root_pkg + " has cargo output files; " + "please rerun with the --copy-out flag." ) with open(CARGO_OUT, "r", encoding="utf-8") as cargo_out: self.parse(cargo_out, "rules.mk") self.crates.sort(key=get_module_name) for crate in self.crates: crate.dump() if self.errors: self.append_to_rules("\n" + ERRORS_LINE + "\n" + self.errors) if self.test_errors: self.append_to_rules( "\n// Errors when listing tests:\n" + self.test_errors ) return self def add_crate(self, crate: Crate): """Append crate to list unless it meets criteria for being skipped.""" if crate.skip_crate(): if self.args.debug: # include debug info of all crates self.crates.append(crate) elif crate.crate_types == set(["bin"]): print("WARNING: skipping binary crate: " + crate.crate_name) else: self.crates.append(crate) def find_warning_owners(self): """For each warning file, find its owner crate.""" missing_owner = False for f in self.warning_files: cargo_dir = "" # find lowest crate, with longest path owner = None # owner crate of this warning for c in self.crates: if f.startswith(c.cargo_dir + "/") and len(cargo_dir) < len( c.cargo_dir ): cargo_dir = c.cargo_dir owner = c if owner: owner.has_warning = True else: missing_owner = True if missing_owner and os.path.exists("Cargo.toml"): # owner is the root cargo, with empty cargo_dir for c in self.crates: if not c.cargo_dir: c.has_warning = True def rustc_command(self, n, rustc_line, line, outf_name): """Process a rustc command line from cargo -vv output.""" # cargo build -vv output can have multiple lines for a rustc command # due to '\n' in strings for environment variables. # strip removes leading spaces and '\n' at the end new_rustc = (rustc_line.strip() + line) if rustc_line else line # Use an heuristic to detect the completions of a multi-line command. # This might fail for some very rare case, but easy to fix manually. if not line.endswith("`\n") or (new_rustc.count("`") % 2) != 0: return new_rustc if match := RUSTC_VV_CMD_ARGS.match(new_rustc): args = match.group(2) self.add_crate(Crate(self, outf_name).parse(n, args)) else: self.assert_empty_vv_line(new_rustc) return "" def append_to_rules(self, line): self.init_rules_file("rules.mk") with open("rules.mk", "a", encoding="utf-8") as outf: outf.write(line) def assert_empty_vv_line(self, line): if line: # report error if line is not empty self.append_to_rules("ERROR -vv line: " + line) return "" def add_empty_test(self, name): if name.startswith("unittests"): self.empty_unittests = True else: self.empty_tests.add(name) def should_ignore_test(self, src): # cargo test outputs the source file for integration tests but # "unittests" for unit tests. To figure out to which crate this # corresponds, we check if the current source file is the main source of # a non-test crate, e.g., a library or a binary. return ( src in self.args.test_blocklist or src in self.empty_tests or ( self.empty_unittests and src in [ c.main_src for c in self.crates if c.crate_types != ["test"] ] ) ) def parse(self, inf, outf_name): """Parse rustc, test, and warning messages in input file.""" n = 0 # line number # We read the file in two passes, where the first simply checks for # empty tests. Otherwise we would add and merge tests before seeing # they're empty. cur_test_name = None for line in inf: if match := CARGO_TEST_LIST_START_PAT.match(line): cur_test_name = match.group(1) elif cur_test_name and ( match := CARGO_TEST_LIST_END_PAT.match(line) ): if int(match.group(1)) + int(match.group(2)) == 0: self.add_empty_test(cur_test_name) cur_test_name = None inf.seek(0) prev_warning = False # true if the previous line was warning: ... rustc_line = "" # previous line(s) matching RUSTC_VV_PAT in_tests = False for line in inf: n += 1 if line.startswith("warning: "): prev_warning = True rustc_line = self.assert_empty_vv_line(rustc_line) continue new_rustc = "" if match := RUSTC_PAT.match(line): args_line = match.group(2) self.add_crate(Crate(self, outf_name).parse(n, args_line)) self.assert_empty_vv_line(rustc_line) elif rustc_line or RUSTC_VV_PAT.match(line): new_rustc = self.rustc_command(n, rustc_line, line, outf_name) elif CC_AR_VV_PAT.match(line): raise NotImplementedError("$CC or $AR commands not supported") elif prev_warning and (match := WARNING_FILE_PAT.match(line)): self.assert_empty_vv_line(rustc_line) fpath = match.group(1) if fpath[0] != "/": # ignore absolute path self.warning_files.add(fpath) elif line.startswith("error: ") or line.startswith("error[E"): if not self.args.ignore_cargo_errors: if in_tests: self.test_errors += "// " + line else: self.errors += line elif CARGO2ANDROID_RUNNING_PAT.match(line): in_tests = "cargo test" in line and "--list" in line prev_warning = False rustc_line = new_rustc self.find_warning_owners() def get_parser(): """Parse main arguments.""" parser = argparse.ArgumentParser("cargo2rulesmk") parser.add_argument( "--add_workspace", action="store_true", default=False, help=( "append [workspace] to Cargo.toml before calling cargo," + " to treat current directory as root of package source;" + " otherwise the relative source file path in generated" + " rules.mk file will be from the parent directory." ), ) parser.add_argument( "--cargo", action="append", metavar="args_string", help=( "extra cargo build -v args in a string, " + "each --cargo flag calls cargo build -v once" ), ) parser.add_argument( "--cargo_bin", type=str, help="use cargo in the cargo_bin directory instead of the prebuilt one", ) parser.add_argument( "--copy-out", action="store_true", default=False, help=( "only for root directory, " + "copy build.rs output to ./out/* and declare source deps " + "for ./out/*.rs; for crates with code pattern: " + 'include!(concat!(env!("OUT_DIR"), "/.rs"))' ), ) parser.add_argument( "--debug", action="store_true", default=False, help="dump debug info into rules.mk", ) parser.add_argument( "--features", type=str, help=( "pass features to cargo build, " + "empty string means no default features" ), ) parser.add_argument( "--ignore-cargo-errors", action="store_true", default=False, help="do not append cargo/rustc error messages to rules.mk", ) parser.add_argument( "--no-subdir", action="store_true", default=False, help="do not output anything for sub-directories", ) parser.add_argument( "--onefile", action="store_true", default=False, help=( "output all into one ./rules.mk, default will generate " + "one rules.mk per Cargo.toml in subdirectories" ), ) parser.add_argument( "--patch", type=str, help="apply the given patch file to generated ./rules.mk", ) parser.add_argument( "--run", action="store_true", default=False, help="run it, default is dry-run", ) parser.add_argument("--rustflags", type=str, help="passing flags to rustc") parser.add_argument( "--skipcargo", action="store_true", default=False, help="skip cargo command, parse cargo.out, and generate ./rules.mk", ) parser.add_argument( "--tests", action="store_true", default=False, help="run cargo build --tests after normal build", ) parser.add_argument( "--use-cargo-lock", action="store_true", default=False, help=( "run cargo build with existing Cargo.lock " + "(used when some latest dependent crates failed)" ), ) parser.add_argument( "--test-data", nargs="*", default=[], help=( "Add the given file to the given test's data property. " + "Usage: test-path=data-path" ), ) parser.add_argument( "--dependency-blocklist", nargs="*", default=[], help="Do not emit the given dependencies (without lib prefixes).", ) parser.add_argument( "--test-blocklist", nargs="*", default=[], help=( "Do not emit the given tests. " + "Pass the path to the test file to exclude." ), ) parser.add_argument( "--cfg-blocklist", nargs="*", default=[], help="Do not emit the given cfg.", ) parser.add_argument( "--verbose", action="store_true", default=False, help="echo executed commands", ) parser.add_argument( "--vv", action="store_true", default=False, help="run cargo with -vv instead of default -v", ) parser.add_argument( "--dump-config-and-exit", type=str, help=( "Dump command-line arguments (minus this flag) to a config file and" " exit. This is intended to help migrate from command line options " "to config files." ), ) parser.add_argument( "--config", type=str, help=( "Load command-line options from the given config file. Options in " "this file will override those passed on the command line." ), ) return parser def parse_args(parser): """Parses command-line options.""" args = parser.parse_args() # Use the values specified in a config file if one was found. if args.config: with open(args.config, "r", encoding="utf-8") as f: config = json.load(f) args_dict = vars(args) for arg in config: args_dict[arg.replace("-", "_")] = config[arg] return args def dump_config(parser, args): """Writes the non-default command-line options to the specified file.""" args_dict = vars(args) # Filter out the arguments that have their default value. # Also filter certain "temporary" arguments. non_default_args = {} for arg in args_dict: if ( args_dict[arg] != parser.get_default(arg) and arg != "dump_config_and_exit" and arg != "config" and arg != "cargo_bin" ): non_default_args[arg.replace("_", "-")] = args_dict[arg] # Write to the specified file. with open(args.dump_config_and_exit, "w", encoding="utf-8") as f: json.dump(non_default_args, f, indent=2, sort_keys=True) def main(): parser = get_parser() args = parse_args(parser) if not args.run: # default is dry-run print(DRY_RUN_NOTE) if args.dump_config_and_exit: dump_config(parser, args) else: Runner(args).run_cargo().gen_rules().apply_patch() if __name__ == "__main__": main()