1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
import json
import os
import pathlib
import platform
import sys
import tempfile
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
Dict,
Hashable,
Iterator,
List,
Mapping,
Optional,
Set,
)
import jsonschema
import py.path
import pytest
import yaml
from _pytest.config.argparsing import Parser
from _pytest.nodes import Node
from pytest_mypy_plugins import utils
if TYPE_CHECKING:
from pytest_mypy_plugins.item import YamlTestItem
@dataclass
class File:
path: str
content: str
def validate_schema(data: Any, *, is_closed: bool = False) -> None:
"""Validate the schema of the file-under-test."""
# Unfortunately, yaml.safe_load() returns Any,
# so we make our intention explicit here.
if not isinstance(data, list):
raise TypeError(f"Test file has to be YAML list, got {type(data)!r}.")
schema = json.loads((pathlib.Path(__file__).parent / "schema.json").read_text("utf8"))
schema["items"]["properties"]["__line__"] = {
"type": "integer",
"description": "Line number where the test starts (`pytest-mypy-plugins` internal)",
}
schema["items"]["additionalProperties"] = not is_closed
jsonschema.validate(instance=data, schema=schema)
def parse_test_files(test_files: List[Dict[str, Any]]) -> List[File]:
files: List[File] = []
for test_file in test_files:
path = test_file.get("path", "main.py")
file = File(path=path, content=test_file.get("content", ""))
files.append(file)
return files
def parse_environment_variables(env_vars: List[str]) -> Dict[str, str]:
parsed_vars: Dict[str, str] = {}
for env_var in env_vars:
name, _, value = env_var.partition("=")
parsed_vars[name] = value
return parsed_vars
def parse_parametrized(params: List[Mapping[str, Any]]) -> List[Mapping[str, Any]]:
if not params:
return [{}]
parsed_params: List[Mapping[str, Any]] = []
known_params: Optional[Set[str]] = None
for idx, param in enumerate(params):
param_keys = set(sorted(param.keys()))
if not known_params:
known_params = param_keys
elif known_params.intersection(param_keys) != known_params:
raise ValueError(
"All parametrized entries must have same keys."
f'First entry is {", ".join(known_params)} but {", ".join(param_keys)} '
"was spotted at {idx} position",
)
parsed_params.append({k: v for k, v in param.items() if not k.startswith("__")})
return parsed_params
class SafeLineLoader(yaml.SafeLoader):
def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> Dict[Hashable, Any]:
mapping = super().construct_mapping(node, deep=deep)
# Add 1 so line numbering starts at 1
starting_line = node.start_mark.line + 1
for title_node, _contents_node in node.value:
if title_node.value == "main":
starting_line = title_node.start_mark.line + 1
mapping["__line__"] = starting_line
return mapping
class YamlTestFile(pytest.File):
def collect(self) -> Iterator["YamlTestItem"]:
from pytest_mypy_plugins.item import YamlTestItem
parsed_file = yaml.load(stream=self.path.read_text("utf8"), Loader=SafeLineLoader)
if parsed_file is None:
return
validate_schema(parsed_file, is_closed=self.config.option.mypy_closed_schema)
if not isinstance(parsed_file, list):
raise ValueError(f"Test file has to be YAML list, got {type(parsed_file)!r}.")
for raw_test in parsed_file:
test_name_prefix = raw_test["case"]
if " " in test_name_prefix:
raise ValueError(f"Invalid test name {test_name_prefix!r}, only '[a-zA-Z0-9_]' is allowed.")
else:
parametrized = parse_parametrized(raw_test.get("parametrized", []))
for params in parametrized:
if params:
test_name_suffix = ",".join(f"{k}={v}" for k, v in params.items())
test_name_suffix = f"[{test_name_suffix}]"
else:
test_name_suffix = ""
test_name = f"{test_name_prefix}{test_name_suffix}"
main_content = utils.render_template(template=raw_test["main"], data=params)
main_file = File(path="main.py", content=main_content)
test_files = [main_file] + parse_test_files(raw_test.get("files", []))
expect_fail = raw_test.get("expect_fail", False)
regex = raw_test.get("regex", False)
expected_output = []
for test_file in test_files:
output_lines = utils.extract_output_matchers_from_comments(
test_file.path, test_file.content.split("\n"), regex=regex
)
expected_output.extend(output_lines)
starting_lineno = raw_test["__line__"]
extra_environment_variables = parse_environment_variables(raw_test.get("env", []))
disable_cache = raw_test.get("disable_cache", False)
expected_output.extend(
utils.extract_output_matchers_from_out(raw_test.get("out", ""), params, regex=regex)
)
additional_mypy_config = utils.render_template(template=raw_test.get("mypy_config", ""), data=params)
skip = self._eval_skip(str(raw_test.get("skip", "False")))
if not skip:
yield YamlTestItem.from_parent(
self,
name=test_name,
files=test_files,
starting_lineno=starting_lineno,
environment_variables=extra_environment_variables,
disable_cache=disable_cache,
expected_output=expected_output,
parsed_test_data=raw_test,
mypy_config=additional_mypy_config,
expect_fail=expect_fail,
)
def _eval_skip(self, skip_if: str) -> bool:
return bool(eval(skip_if, {"sys": sys, "os": os, "pytest": pytest, "platform": platform}))
def pytest_collect_file(file_path: pathlib.Path, parent: Node) -> Optional[YamlTestFile]:
if file_path.suffix in {".yaml", ".yml"} and file_path.name.startswith(("test-", "test_")):
return YamlTestFile.from_parent(parent, path=file_path, fspath=None)
return None
def pytest_addoption(parser: Parser) -> None:
group = parser.getgroup("mypy-tests")
group.addoption(
"--mypy-testing-base", type=str, default=tempfile.gettempdir(), help="Base directory for tests to use"
)
group.addoption(
"--mypy-pyproject-toml-file",
type=str,
help="Which `pyproject.toml` file to use as a default config for tests. Incompatible with `--mypy-ini-file`",
)
group.addoption(
"--mypy-ini-file",
type=str,
help="Which `.ini` file to use as a default config for tests. Incompatible with `--mypy-pyproject-toml-file`",
)
group.addoption(
"--mypy-same-process",
action="store_true",
help="Run in the same process. Useful for debugging, will create problems with import cache",
)
group.addoption(
"--mypy-extension-hook",
type=str,
help="Fully qualified path to the extension hook function, in case you need custom yaml keys. "
"Has to be top-level.",
)
group.addoption(
"--mypy-only-local-stub",
action="store_true",
help="mypy will ignore errors from site-packages",
)
group.addoption(
"--mypy-closed-schema",
action="store_true",
help="Use closed schema to validate YAML test cases, which won't allow any extra keys (does not work well with `--mypy-extension-hook`)",
)
|
