1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
#!/usr/bin/env python
"""Check that Python.h is included before any stdlib headers.
May be a bit overzealous, but it should get the job done.
"""
import argparse
import fnmatch
import os.path
import re
import subprocess
import sys
from get_submodule_paths import get_submodule_paths
HEADER_PATTERN = re.compile(
r'^\s*#\s*include\s*[<"]((?:\w+/)*\w+(?:\.h[hp+]{0,2})?)[>"]\s*$'
)
PYTHON_INCLUDING_HEADERS = [
"Python.h",
# This isn't all of Python.h, but it is the visibility macros
"pyconfig.h",
"numpy/arrayobject.h",
"numpy/ndarrayobject.h",
"numpy/npy_common.h",
"numpy/npy_math.h",
"numpy/random/distributions.h",
"pybind11/pybind11.h",
# Boost::Python
"boost/python.hpp",
"boost/python/args.hpp",
"boost/python/detail/prefix.hpp",
"boost/python/detail/wrap_python.hpp",
"boost/python/ssize_t.hpp",
"boost/python/object.hpp",
"boost/mpi/python.hpp",
# Pythran
"pythonic/core.hpp",
# Python-including headers the sort doesn't pick up
"ni_support.h",
# xsf::numpy
"xsf/numpy.h",
]
LEAF_HEADERS = []
C_CPP_EXTENSIONS = (".c", ".h", ".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx")
# check against list in diff_files
PARSER = argparse.ArgumentParser(description=__doc__)
PARSER.add_argument(
"--diff-against",
dest="branch",
type=str,
default=None,
help="Diff against "
"this branch and lint modified files. Use either "
"`--diff-against` or `--files`, but not both. "
"Likely to produce false positives.",
)
PARSER.add_argument(
"files",
nargs="*",
help="Lint these files or directories; " "use **/*.py to lint all files",
)
def check_python_h_included_first(name_to_check: str) -> int:
"""Check that the passed file includes Python.h first if it does at all.
Perhaps overzealous, but that should work around concerns with
recursion.
Parameters
----------
name_to_check : str
The name of the file to check.
Returns
-------
int
The number of headers before Python.h
"""
included_python = False
included_non_python_header = []
warned_python_construct = False
basename_to_check = os.path.basename(name_to_check)
in_comment = False
includes_headers = False
with open(name_to_check) as in_file:
for i, line in enumerate(in_file, 1):
# Very basic comment parsing
# Assumes /*...*/ comments are on their own lines
if "/*" in line:
if "*/" not in line:
in_comment = True
# else-branch could use regex to remove comment and continue
continue
if in_comment:
if "*/" in line:
in_comment = False
continue
match = HEADER_PATTERN.match(line)
if match:
includes_headers = True
this_header = match.group(1)
if this_header in PYTHON_INCLUDING_HEADERS:
if included_non_python_header and not included_python:
print(
f"Header before Python.h in file {name_to_check:s}\n"
f"Python.h on line {i:d}, other header(s) on line(s)"
f" {included_non_python_header}",
file=sys.stderr,
)
included_python = True
PYTHON_INCLUDING_HEADERS.append(basename_to_check)
elif not included_python and (
"numpy" in this_header
and this_header != "numpy/utils.h"
or "python" in this_header
):
print(
f"Python.h not included before python-including header "
f"in file {name_to_check:s}\n"
f"{this_header:s} on line {i:d}",
file=sys.stderr,
)
elif not included_python and this_header not in LEAF_HEADERS:
included_non_python_header.append(i)
elif (
not included_python
and not warned_python_construct
and ".h" not in basename_to_check
) and ("py::" in line or "PYBIND11_" in line or "npy_" in line):
print(
"Python-including header not used before python constructs "
f"in file {name_to_check:s}\nConstruct on line {i:d}",
file=sys.stderr,
)
warned_python_construct = True
if includes_headers:
LEAF_HEADERS.append(this_header)
return included_python and len(included_non_python_header)
def process_files(file_list: list[str]) -> int:
n_out_of_order = 0
submodule_paths = get_submodule_paths()
root_directory = os.path.dirname(os.path.dirname(__file__))
for name_to_check in sorted(
file_list, key=lambda name: "h" not in os.path.splitext(name)[1].lower()
):
name_to_check = os.path.join(root_directory, name_to_check)
if any(submodule_path in name_to_check for submodule_path in submodule_paths):
continue
try:
n_out_of_order += check_python_h_included_first(name_to_check)
except UnicodeDecodeError:
print(f"File {name_to_check:s} not utf-8", sys.stdout)
return n_out_of_order
def find_c_cpp_files(root: str) -> list[str]:
result = []
for dirpath, dirnames, filenames in os.walk("scipy"):
# I'm assuming other people have checked boost
for name in ("build", ".git", "boost"):
try:
dirnames.remove(name)
except ValueError:
pass
for name in fnmatch.filter(dirnames, "*.p"):
dirnames.remove(name)
result.extend(
[
os.path.join(dirpath, name)
for name in filenames
if os.path.splitext(name)[1].lower() in C_CPP_EXTENSIONS
]
)
return result
def diff_files(sha: str) -> list[str]:
"""Find the diff since the given SHA.
Adapted from lint.py
"""
res = subprocess.run(
[
"git",
"diff",
"--name-only",
"--diff-filter=ACMR",
"-z",
sha,
"--",
# Check against C_CPP_EXTENSIONS
"*.[chCH]",
"*.[ch]pp",
"*.[ch]xx",
"*.cc",
"*.hh",
],
stdout=subprocess.PIPE,
encoding="utf-8",
)
res.check_returncode()
return [f for f in res.stdout.split("\0") if f]
if __name__ == "__main__":
from lint import find_branch_point
args = PARSER.parse_args()
if not ((len(args.files) == 0) ^ (args.branch is None)):
files = find_c_cpp_files("scipy")
elif args.branch:
branch_point = find_branch_point(args.branch)
files = diff_files(branch_point)
else:
files = args.files
# See which of the headers include Python.h and add them to the list
n_out_of_order = process_files(files)
sys.exit(n_out_of_order)
|
