1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
#!/usr/bin/env python3
from __future__ import annotations
import json
import os
import re
import subprocess
import sys
from pathlib import Path
try:
from termcolor import colored
except ImportError:
def colored(text: str, color: str = "") -> str: # type: ignore[misc]
return text
_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
_SUCCESS = colored("Success", "green")
_SKIPPED = colored("Skipped", "yellow")
_FAILED = colored("Failed", "red")
# We're using the oldest supported version because it's the most likely to produce errors
# due to unsupported syntax, feature, or bug in a tool.
_PYTHON_VERSION = "3.7"
def _parse_jsonc(json_text: str) -> str:
# strip comments from the file
lines = [line for line in json_text.split("\n") if not line.strip().startswith("//")]
# strip trailing commas from the file
valid_json = re.sub(r",(\s*?[\}\]])", r"\1", "\n".join(lines))
return valid_json
def _get_strict_params(stub_path: str) -> list[str]:
with open(_STRICTER_CONFIG_FILE, encoding="UTF-8") as file:
data = json.loads(_parse_jsonc(file.read()))
lower_stub_path = stub_path.lower()
if any(lower_stub_path == stub.lower() for stub in data["exclude"]):
return []
return ["-p", _STRICTER_CONFIG_FILE]
def main() -> None:
try:
path = sys.argv[1]
except IndexError:
print("Missing path argument in format <folder>/<stub>", file=sys.stderr)
sys.exit(1)
assert os.path.exists(path), rf"Path {path} does not exist."
path_tokens = Path(path).parts
assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>."
folder, stub = path_tokens
assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported."
stubtest_result: subprocess.CompletedProcess[bytes] | None = None
pytype_result: subprocess.CompletedProcess[bytes] | None = None
# Run formatters first. Order matters.
print("\nRunning pycln...")
subprocess.run([sys.executable, "-m", "pycln", path, "--all"])
print("\nRunning isort...")
subprocess.run([sys.executable, "-m", "isort", path])
print("\nRunning Black...")
black_result = subprocess.run([sys.executable, "-m", "black", path])
if black_result.returncode == 123:
print("Could not run tests due to an internal error with Black. See above for details.", file=sys.stderr)
sys.exit(black_result.returncode)
print("\nRunning Flake8...")
flake8_result = subprocess.run([sys.executable, "-m", "flake8", path])
print("\nRunning check_consistent.py...")
check_consistent_result = subprocess.run([sys.executable, "tests/check_consistent.py"])
print("\nRunning check_new_syntax.py...")
check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
print(f"\nRunning Pyright on Python {_PYTHON_VERSION}...")
pyright_result = subprocess.run(
[sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + _get_strict_params(path),
stderr=subprocess.PIPE,
text=True,
)
if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
print(colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow"))
pyright_returncode = 0
pyright_skipped = True
else:
print(pyright_result.stderr)
pyright_returncode = pyright_result.returncode
pyright_skipped = False
print(f"\nRunning mypy for Python {_PYTHON_VERSION}...")
mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", _PYTHON_VERSION])
# If mypy failed, stubtest will fail without any helpful error
if mypy_result.returncode == 0:
if folder == "stdlib":
print("\nRunning stubtest...")
stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])
else:
run_stubtest_query = (
f"\nRun stubtest for {stub!r} (Y/N)?\n\n"
"NOTE: Running third-party stubtest involves downloading and executing arbitrary code from PyPI.\n"
f"Only run stubtest if you trust the {stub!r} package.\n"
)
run_stubtest_answer = input(colored(run_stubtest_query, "yellow")).lower()
while run_stubtest_answer not in {"yes", "no", "y", "n"}:
run_stubtest_answer = input(colored("Invalid response; please try again.\n", "red")).lower()
if run_stubtest_answer in {"yes", "y"}:
print("\nRunning stubtest.")
stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
else:
print(colored(f"\nSkipping stubtest for {stub!r}...", "yellow"))
else:
print(colored("\nSkipping stubtest since mypy failed.", "yellow"))
if sys.platform == "win32":
print(colored("\nSkipping pytype on Windows. You can run the test with WSL.", "yellow"))
else:
print("\nRunning pytype...")
pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
print(f"\nRunning regression tests for Python {_PYTHON_VERSION}...")
regr_test_result = subprocess.run(
[sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
stderr=subprocess.PIPE,
text=True,
)
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
if "No test cases found" in regr_test_result.stderr:
regr_test_returncode = 0
print(colored(f"\nNo test cases found for {stub!r}!", "green"))
else:
regr_test_returncode = regr_test_result.returncode
print(regr_test_result.stderr)
any_failure = any(
[
flake8_result.returncode,
check_consistent_result.returncode,
check_new_syntax_result.returncode,
pyright_returncode,
mypy_result.returncode,
getattr(stubtest_result, "returncode", 0),
getattr(pytype_result, "returncode", 0),
regr_test_returncode,
]
)
if any_failure:
print(colored("\n\n--- TEST SUMMARY: One or more tests failed. See above for details. ---\n", "red"))
else:
print(colored("\n\n--- TEST SUMMARY: All tests passed! ---\n", "green"))
print("Flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
print("Check consistent:", _SUCCESS if check_consistent_result.returncode == 0 else _FAILED)
print("Check new syntax:", _SUCCESS if check_new_syntax_result.returncode == 0 else _FAILED)
if pyright_skipped:
print("Pyright:", _SKIPPED)
else:
print("Pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
if stubtest_result is None:
print("stubtest:", _SKIPPED)
else:
print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
if pytype_result is None:
print("pytype:", _SKIPPED)
else:
print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
sys.exit(int(any_failure))
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print(colored("\nTests aborted due to KeyboardInterrupt!\n", "red"))
sys.exit(1)
|
