#!/usr/bin/env python3 # pylint: disable=protected-access, unused-variable, locally-disabled, len-as-condition """Lint helper to generate lint summary of source. Copyright by Contributors """ from __future__ import print_function import argparse import codecs import sys import re import os import cpplint from cpplint import _cpplint_state from pylint import epylint CXX_SUFFIX = set(['cc', 'c', 'cpp', 'h', 'cu', 'hpp']) PYTHON_SUFFIX = set(['py']) def filepath_enumerate(paths): """Enumerate the file paths of all subfiles of the list of paths""" out = [] for path in paths: if os.path.isfile(path): out.append(path) else: for root, dirs, files in os.walk(path): for name in files: out.append(os.path.normpath(os.path.join(root, name))) return out # pylint: disable=useless-object-inheritance class LintHelper(object): """Class to help runing the lint and records summary""" @staticmethod def _print_summary_map(strm, result_map, ftype): """Print summary of certain result map.""" if len(result_map) == 0: return 0 npass = sum(1 for x in result_map.values() if len(x) == 0) strm.write(f'====={npass}/{len(result_map)} {ftype} files passed check=====\n') for fname, emap in result_map.items(): if len(emap) == 0: continue strm.write( f'{fname}: {sum(emap.values())} Errors of {len(emap)} Categories map={str(emap)}\n') return len(result_map) - npass def __init__(self): self.project_name = None self.cpp_header_map = {} self.cpp_src_map = {} self.python_map = {} pylint_disable = ['superfluous-parens', 'too-many-instance-attributes', 'too-few-public-methods'] # setup pylint self.pylint_opts = ['--extension-pkg-whitelist=numpy', '--disable=' + ','.join(pylint_disable)] self.pylint_cats = set(['error', 'warning', 'convention', 'refactor']) # setup cpp lint cpplint_args = ['--quiet', '--extensions=' + (','.join(CXX_SUFFIX)), '.'] _ = cpplint.ParseArguments(cpplint_args) cpplint._SetFilters(','.join(['-build/c++11', '-build/namespaces', '-build/include,', '+build/include_what_you_use', '+build/include_order'])) cpplint._SetCountingStyle('toplevel') cpplint._line_length = 100 def process_cpp(self, path, suffix): """Process a cpp file.""" _cpplint_state.ResetErrorCounts() cpplint.ProcessFile(str(path), _cpplint_state.verbose_level) _cpplint_state.PrintErrorCounts() errors = _cpplint_state.errors_by_category.copy() if suffix == 'h': self.cpp_header_map[str(path)] = errors else: self.cpp_src_map[str(path)] = errors def process_python(self, path): """Process a python file.""" (pylint_stdout, pylint_stderr) = epylint.py_run( ' '.join([str(path)] + self.pylint_opts), return_std=True) emap = {} err = pylint_stderr.read() if len(err): print(err) for line in pylint_stdout: sys.stderr.write(line) key = line.split(':')[-1].split('(')[0].strip() if key not in self.pylint_cats: continue if key not in emap: emap[key] = 1 else: emap[key] += 1 self.python_map[str(path)] = emap def print_summary(self, strm): """Print summary of lint.""" nerr = 0 nerr += LintHelper._print_summary_map(strm, self.cpp_header_map, 'cpp-header') nerr += LintHelper._print_summary_map(strm, self.cpp_src_map, 'cpp-source') nerr += LintHelper._print_summary_map(strm, self.python_map, 'python') if nerr == 0: strm.write('All passed!\n') else: strm.write(f'{nerr} files failed lint\n') return nerr # singleton helper for lint check _HELPER = LintHelper() def get_header_guard_dmlc(filename): """Get Header Guard Convention for DMLC Projects. For headers in include, directly use the path For headers in src, use project name plus path Examples: with project-name = dmlc include/dmlc/timer.h -> DMLC_TIMTER_H_ src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_ """ fileinfo = cpplint.FileInfo(filename) file_path_from_root = fileinfo.RepositoryName() inc_list = ['include', 'api', 'wrapper', 'contrib'] if os.name == 'nt': inc_list.append("mshadow") if file_path_from_root.find('src/') != -1 and _HELPER.project_name is not None: idx = file_path_from_root.find('src/') file_path_from_root = _HELPER.project_name + file_path_from_root[idx + 3:] else: idx = file_path_from_root.find("include/") if idx != -1: file_path_from_root = file_path_from_root[idx + 8:] for spath in inc_list: prefix = spath + '/' if file_path_from_root.startswith(prefix): file_path_from_root = re.sub('^' + prefix, '', file_path_from_root) break return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_' cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc def process(fname, allow_type): """Process a file.""" fname = str(fname) arr = fname.rsplit('.', 1) if fname.find('#') != -1 or arr[-1] not in allow_type: return if arr[-1] in CXX_SUFFIX: _HELPER.process_cpp(fname, arr[-1]) if arr[-1] in PYTHON_SUFFIX: _HELPER.process_python(fname) def main(): """Main entry function.""" parser = argparse.ArgumentParser(description="lint source codes") parser.add_argument('project', help='project name') parser.add_argument('filetype', choices=['python', 'cpp', 'all'], help='source code type') parser.add_argument('path', nargs='+', help='path to traverse') parser.add_argument('--exclude_path', nargs='+', default=[], help='exclude this path, and all subfolders if path is a folder') parser.add_argument('--quiet', action='store_true', help='run cpplint in quiet mode') parser.add_argument('--pylint-rc', default=None, help='pylint rc file') args = parser.parse_args() _HELPER.project_name = args.project if args.pylint_rc is not None: _HELPER.pylint_opts = ['--rcfile='+args.pylint_rc,] file_type = args.filetype allow_type = [] if file_type in ('python', 'all'): allow_type += PYTHON_SUFFIX if file_type in ('cpp', 'all'): allow_type += CXX_SUFFIX allow_type = set(allow_type) if sys.version_info.major == 2 and os.name != 'nt': sys.stderr = codecs.StreamReaderWriter(sys.stderr, codecs.getreader('utf8'), codecs.getwriter('utf8'), 'replace') # get excluded files excluded_paths = filepath_enumerate(args.exclude_path) for path in args.path: if os.path.isfile(path): normpath = os.path.normpath(path) if normpath not in excluded_paths: process(path, allow_type) else: for root, dirs, files in os.walk(path): for name in files: file_path = os.path.normpath(os.path.join(root, name)) if file_path not in excluded_paths: process(file_path, allow_type) nerr = _HELPER.print_summary(sys.stderr) sys.exit(nerr > 0) if __name__ == '__main__': main()