1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
|
# test_source.py -- Tests for scanning dulwich source code
# Copyright (C) 2025 Jelmer Vernooij <jelmer@jelmer.uk>
#
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
# General Public License as published by the Free Software Foundation; version 2.0
# or (at your option) any later version. You can redistribute it and/or
# modify it under the terms of either of these two licenses.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# You should have received a copy of the licenses; if not, see
# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
# License, Version 2.0.
#
"""Tests for scanning dulwich source code for compliance."""
import ast
import os
import re
import unittest
from pathlib import Path
# Files that are allowed to not have the standard preamble
PREAMBLE_EXCEPTIONS = [
"dulwich/diffstat.py", # MIT licensed file
]
# Files that are allowed to use os.environ (beyond cli.py and porcelain/)
OS_ENVIRON_EXCEPTIONS = [
"dulwich/client.py", # Git protocol environment variables
"dulwich/repo.py", # User identity environment variables
"dulwich/log_utils.py", # GIT_TRACE environment variable
"dulwich/config.py", # Git configuration environment variables
"dulwich/gc.py", # GIT_AUTO_GC environment variable
"dulwich/contrib/swift.py", # DULWICH_SWIFT_CFG environment variable
"dulwich/hooks.py", # Git hooks environment setup
]
# Standard license block that must appear in all files
STANDARD_LICENSE_BLOCK = [
"# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later\n",
"# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU\n",
"# General Public License as published by the Free Software Foundation; version 2.0\n",
"# or (at your option) any later version. You can redistribute it and/or\n",
"# modify it under the terms of either of these two licenses.\n",
"#\n",
"# Unless required by applicable law or agreed to in writing, software\n",
'# distributed under the License is distributed on an "AS IS" BASIS,\n',
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
"# See the License for the specific language governing permissions and\n",
"# limitations under the License.\n",
"#\n",
"# You should have received a copy of the licenses; if not, see\n",
"# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License\n",
"# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache\n",
"# License, Version 2.0.\n",
"#\n",
]
def _get_python_files(directory_name):
"""Get all Python files in a directory.
Args:
directory_name: Name of directory relative to project root (e.g., "dulwich", "tests")
Returns:
List of tuples of (Path object, relative path from project root)
"""
project_root = Path(__file__).parent.parent
target_dir = project_root / directory_name
if not target_dir.exists():
raise RuntimeError(f"{directory_name} directory not found at {target_dir}")
python_files = []
for root, dirs, files in os.walk(target_dir):
# Skip build directories
if root.endswith(("build", "__pycache__")):
continue
for file in files:
if file.endswith(".py"):
file_path = Path(root) / file
rel_path = file_path.relative_to(project_root)
python_files.append((file_path, rel_path))
return python_files
def _imports_module(file_path, module_name):
"""Check if a Python file imports a specific module or any submodules.
Args:
file_path: Path to the Python file
module_name: Module name to check for (e.g., "dulwich.porcelain", "dulwich.cli")
Returns:
bool: True if the file imports the module or any submodule
"""
with open(file_path, encoding="utf-8") as f:
tree = ast.parse(f.read(), filename=str(file_path))
for node in ast.walk(tree):
# Check "import dulwich.porcelain" or "import dulwich.porcelain.lfs"
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name == module_name or alias.name.startswith(
f"{module_name}."
):
return True
# Check "from dulwich.porcelain import ..." or "from dulwich import porcelain"
if isinstance(node, ast.ImportFrom):
# "from dulwich.porcelain import something"
# "from dulwich.porcelain.lfs import something"
if node.module == module_name or (
node.module and node.module.startswith(f"{module_name}.")
):
return True
# Handle "from dulwich import porcelain"
if node.module and module_name.startswith(f"{node.module}."):
# e.g., module="dulwich", module_name="dulwich.porcelain"
suffix = module_name[len(node.module) + 1 :]
for alias in node.names:
if alias.name == suffix:
return True
return False
class SourceCodeComplianceTests(unittest.TestCase):
"""Tests to ensure dulwich source code follows project standards."""
@staticmethod
def _get_dulwich_python_files():
"""Get all Python files in the dulwich package.
Returns:
List of tuples of (Path object, relative path from project root)
"""
return _get_python_files("dulwich")
@classmethod
def _has_standard_preamble(cls, file_path: Path) -> tuple[bool, str]:
"""Check if a file has the standard dulwich preamble.
The standard preamble consists of:
- First line: # filename -- Description (or similar)
- Copyright line(s): # Copyright (C) ...
- Empty comment: #
- Standard license block (exact match required)
Args:
file_path: Path to the Python file to check
Returns:
Tuple of (has_preamble, error_message)
"""
with open(file_path, encoding="utf-8") as f:
lines = f.readlines()
if len(lines) < 21:
return False, "File too short to contain standard preamble"
# Check first line starts with #
if not lines[0].startswith("#"):
return False, "First line does not start with #"
# Find the SPDX line (should be within first 10 lines)
spdx_line_idx = None
for i in range(min(10, len(lines))):
if "SPDX-License-Identifier" in lines[i]:
spdx_line_idx = i
break
if spdx_line_idx is None:
return False, "SPDX-License-Identifier line not found in first 10 lines"
# Check that we have enough lines after the SPDX line
if len(lines) < spdx_line_idx + len(STANDARD_LICENSE_BLOCK):
return (
False,
"File too short to contain complete license block after SPDX line",
)
# Extract the license block from the file
file_license_block = lines[
spdx_line_idx : spdx_line_idx + len(STANDARD_LICENSE_BLOCK)
]
# Compare with standard license block
for i, (expected, actual) in enumerate(
zip(STANDARD_LICENSE_BLOCK, file_license_block)
):
if expected != actual:
return (
False,
f"License block mismatch at line {spdx_line_idx + i + 1}: expected {expected!r}, got {actual!r}",
)
return True, ""
def test_all_files_have_preamble(self):
"""Test that all dulwich Python files have the standard preamble."""
python_files = self._get_dulwich_python_files()
self.assertGreater(len(python_files), 0, "No Python files found in dulwich/")
files_without_preamble = []
for file_path, rel_path in python_files:
# Convert to forward slashes for consistency
rel_path_str = str(rel_path).replace(os.sep, "/")
# Skip exceptions
if rel_path_str in PREAMBLE_EXCEPTIONS:
continue
has_preamble, error_msg = self._has_standard_preamble(file_path)
if not has_preamble:
files_without_preamble.append(f"{rel_path_str}: {error_msg}")
if files_without_preamble:
self.fail(
"The following files are missing the standard preamble:\n"
+ "\n".join(f" - {f}" for f in files_without_preamble)
)
def test_os_environ_usage_restricted(self):
"""Test that os.environ is only used in allowed files."""
python_files = self._get_dulwich_python_files()
self.assertGreater(len(python_files), 0, "No Python files found in dulwich/")
# Files allowed to use os.environ
allowed_files = {
"dulwich/cli.py",
"dulwich/porcelain/",
}
# Add exception files
allowed_files.update(OS_ENVIRON_EXCEPTIONS)
files_with_violations = []
# Pattern to match os.environ usage
os_environ_pattern = re.compile(r"\bos\.environ\b")
for file_path, rel_path in python_files:
# Convert to forward slashes for consistency
rel_path_str = str(rel_path).replace(os.sep, "/")
# Skip allowed files
if any(rel_path_str.startswith(f) for f in allowed_files):
continue
with open(file_path, encoding="utf-8") as f:
content = f.read()
matches = os_environ_pattern.findall(content)
if matches:
# Count occurrences
line_numbers = []
for line_num, line in enumerate(content.split("\n"), 1):
if os_environ_pattern.search(line):
line_numbers.append(line_num)
files_with_violations.append(
f"{rel_path_str}: os.environ used on line(s) {', '.join(map(str, line_numbers))}"
)
if files_with_violations:
self.fail(
"The following files use os.environ but are not in the allowed list:\n"
+ "\n".join(f" - {f}" for f in files_with_violations)
+ "\n\nFiles allowed to use os.environ:\n"
+ "\n".join(f" - {f}" for f in sorted(allowed_files))
)
def test_porcelain_usage_restricted_in_tests(self):
"""Test that dulwich.porcelain is only used in allowed test directories."""
test_files = _get_python_files("tests")
self.assertGreater(len(test_files), 0, "No Python files found in tests/")
# Directories allowed to use porcelain
allowed_dirs = {
"tests/cli/",
"tests/porcelain/",
"tests/compat/",
}
# Individual test files allowed to use porcelain
allowed_files: set[str] = set()
files_with_violations = []
for file_path, rel_path in test_files:
# Convert to forward slashes for consistency
rel_path_str = str(rel_path).replace(os.sep, "/")
# Skip allowed directories
if any(rel_path_str.startswith(d) for d in allowed_dirs):
continue
# Skip allowed files
if rel_path_str in allowed_files:
continue
if _imports_module(file_path, "dulwich.porcelain"):
files_with_violations.append(rel_path_str)
if files_with_violations:
self.fail(
"The following test files use dulwich.porcelain but are not in the allowed list:\n"
+ "\n".join(f" - {f}" for f in files_with_violations)
+ "\n\nLower-level tests should use dulwich APIs directly, not porcelain."
+ "\n\nAllowed directories:\n"
+ "\n".join(f" - {d}" for d in sorted(allowed_dirs))
+ "\nAllowed files:\n"
+ "\n".join(f" - {f}" for f in sorted(allowed_files))
)
def test_cli_usage_restricted_in_tests(self):
"""Test that dulwich.cli is only used in CLI test directory."""
test_files = _get_python_files("tests")
self.assertGreater(len(test_files), 0, "No Python files found in tests/")
# Only CLI tests should import dulwich.cli
allowed_dir = "tests/cli/"
files_with_violations = []
for file_path, rel_path in test_files:
# Convert to forward slashes for consistency
rel_path_str = str(rel_path).replace(os.sep, "/")
# Skip allowed directory
if rel_path_str.startswith(allowed_dir):
continue
if _imports_module(file_path, "dulwich.cli"):
files_with_violations.append(rel_path_str)
if files_with_violations:
self.fail(
"The following test files use dulwich.cli but are not in tests/cli/:\n"
+ "\n".join(f" - {f}" for f in files_with_violations)
+ "\n\nOnly CLI tests in tests/cli/ should import dulwich.cli."
)
|
