1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
|
import re
import os
import itertools
from collections import defaultdict
from typing import (Any, Dict, Iterable, List, MutableMapping, Optional, Pattern, Tuple, TypeVar,
Union, cast)
T = TypeVar('T')
end_space = re.compile(r"([^\\]\s)*$")
def fnmatch_translate(pat: bytes) -> Tuple[bool, Pattern[bytes]]:
parts = []
seq: Optional[int] = None
i = 0
any_char = b"[^/]"
if pat[0:1] == b"/":
parts.append(b"^")
pat = pat[1:]
else:
# By default match the entire path up to a /
# but if / doesn't appear in the pattern we will mark is as
# a name pattern and just produce a pattern that matches against
# the filename
parts.append(b"^(?:.*/)?")
name_pattern = True
if pat[-1:] == b"/":
# If the last character is / match this directory or any subdirectory
pat = pat[:-1]
suffix = b"(?:/|$)"
else:
suffix = b"$"
while i < len(pat):
c = pat[i:i+1]
if c == b"\\":
if i < len(pat) - 1:
i += 1
c = pat[i:i+1]
parts.append(re.escape(c))
else:
raise ValueError
elif seq is not None:
# TODO: this doesn't really handle invalid sequences in the right way
if c == b"]":
seq = None
if parts[-1] == b"[":
parts = parts[:-1]
elif parts[-1] == b"^" and parts[-2] == b"[":
raise ValueError
else:
parts.append(c)
elif c == b"-":
parts.append(c)
elif c == b"[":
raise ValueError
else:
parts.append(re.escape(c))
elif c == b"[":
parts.append(b"[")
if i < len(pat) - 1 and pat[i+1:i+2] in (b"!", b"^"):
parts.append(b"^")
i += 1
seq = i
elif c == b"*":
if i < len(pat) - 1 and pat[i+1:i+2] == b"*":
if i > 0 and pat[i-1:i] != b"/":
raise ValueError
parts.append(b".*")
i += 1
if i < len(pat) - 1 and pat[i+1:i+2] != b"/":
raise ValueError
else:
parts.append(any_char + b"*")
elif c == b"?":
parts.append(any_char)
elif c == b"/" and not seq:
name_pattern = False
parts.append(c)
else:
parts.append(re.escape(c))
i += 1
if name_pattern:
parts[0] = b"^"
if seq is not None:
raise ValueError
parts.append(suffix)
try:
return name_pattern, re.compile(b"".join(parts))
except Exception:
raise ValueError
# Regexp matching rules that have to be converted to patterns
pattern_re = re.compile(br".*[\*\[\?]")
def parse_line(line: bytes) -> Optional[Tuple[bool, bool, bool, Union[Tuple[bytes, ...], Tuple[bool, Pattern[bytes]]]]]:
line = line.rstrip()
if not line or line[0:1] == b"#":
return None
invert = line[0:1] == b"!"
if invert:
line = line[1:]
dir_only = line[-1:] == b"/"
if dir_only:
line = line[:-1]
# Could make a special case for **/foo, but we don't have any patterns like that
if not invert and not pattern_re.match(line):
literal = True
pattern: Union[Tuple[bytes, ...], Tuple[bool, Pattern[bytes]]] = tuple(line.rsplit(b"/", 1))
else:
pattern = fnmatch_translate(line)
literal = False
return invert, dir_only, literal, pattern
class PathFilter:
def __init__(self, root: bytes, extras: Optional[List[bytes]] = None, cache: Optional[MutableMapping[bytes, bool]] = None) -> None:
if root:
ignore_path: Optional[bytes] = os.path.join(root, b".gitignore")
else:
ignore_path = None
if not ignore_path and not extras:
self.trivial = True
return
self.trivial = False
self.literals_file: Dict[Optional[bytes], Dict[bytes, List[Tuple[bool, Pattern[bytes]]]]] = defaultdict(dict)
self.literals_dir: Dict[Optional[bytes], Dict[bytes, List[Tuple[bool, Pattern[bytes]]]]] = defaultdict(dict)
self.patterns_file: List[Tuple[Tuple[bool, Pattern[bytes]], List[Tuple[bool, Pattern[bytes]]]]] = []
self.patterns_dir: List[Tuple[Tuple[bool, Pattern[bytes]], List[Tuple[bool, Pattern[bytes]]]]] = []
if cache is None:
cache = {}
self.cache: MutableMapping[bytes, bool] = cache
if extras is None:
extras = []
if ignore_path and os.path.exists(ignore_path):
args: Tuple[Optional[bytes], List[bytes]] = (ignore_path, extras)
else:
args = None, extras
self._read_ignore(*args)
def _read_ignore(self, ignore_path: Optional[bytes], extras: List[bytes]) -> None:
if ignore_path is not None:
with open(ignore_path, "rb") as f:
for line in f:
self._read_line(line)
for line in extras:
self._read_line(line)
def _read_line(self, line: bytes) -> None:
parsed = parse_line(line)
if not parsed:
return
invert, dir_only, literal, rule = parsed
if invert:
# For exclude rules, we attach the rules to all preceeding patterns, so
# that we can match patterns out of order and check if they were later
# overridden by an exclude rule
assert not literal
rule = cast(Tuple[bool, Pattern[bytes]], rule)
if not dir_only:
rules_iter: Iterable[Tuple[Any, List[Tuple[bool, Pattern[bytes]]]]] = itertools.chain(
itertools.chain(*(item.items() for item in self.literals_dir.values())),
itertools.chain(*(item.items() for item in self.literals_file.values())),
self.patterns_dir,
self.patterns_file)
else:
rules_iter = itertools.chain(
itertools.chain(*(item.items() for item in self.literals_dir.values())),
self.patterns_dir)
for rules in rules_iter:
rules[1].append(rule)
else:
if literal:
rule = cast(Tuple[bytes, ...], rule)
if len(rule) == 1:
dir_name, pattern = None, rule[0] # type: Tuple[Optional[bytes], bytes]
else:
dir_name, pattern = rule
self.literals_dir[dir_name][pattern] = []
if not dir_only:
self.literals_file[dir_name][pattern] = []
else:
rule = cast(Tuple[bool, Pattern[bytes]], rule)
self.patterns_dir.append((rule, []))
if not dir_only:
self.patterns_file.append((rule, []))
def filter(self,
iterator: Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]
) -> Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]:
empty: Dict[Any, Any] = {}
for dirpath, dirnames, filenames in iterator:
orig_dirpath = dirpath
path_sep = os.path.sep.encode()
if path_sep != b"/":
dirpath = dirpath.replace(path_sep, b"/")
keep_dirs: List[Tuple[bytes, T]] = []
keep_files: List[Tuple[bytes, T]] = []
for iter_items, literals, patterns, target, suffix in [
(dirnames, self.literals_dir, self.patterns_dir, keep_dirs, b"/"),
(filenames, self.literals_file, self.patterns_file, keep_files, b"")]:
for item in iter_items:
name = item[0]
if dirpath:
path = b"%s/%s" % (dirpath, name) + suffix
else:
path = name + suffix
if path in self.cache:
if not self.cache[path]:
target.append(item)
continue
for rule_dir in [None, dirpath if dirpath != b"." else b""]:
if name in literals.get(rule_dir, empty):
exclude = literals[rule_dir][name]
if not any(rule.match(name if name_only else path)
for name_only, rule in exclude):
# Skip this item
self.cache[path] = True
break
else:
for (component_only, pattern), exclude in patterns:
if component_only:
match = pattern.match(name)
else:
match = pattern.match(path)
if match:
if not any(rule.match(name if name_only else path)
for name_only, rule in exclude):
# Skip this item
self.cache[path] = True
break
else:
self.cache[path] = False
target.append(item)
dirnames[:] = keep_dirs
assert not any(b".git" == name for name, _ in dirnames)
yield orig_dirpath, dirnames, keep_files
def __call__(self,
iterator: Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]
) -> Iterable[Tuple[bytes, List[Tuple[bytes, T]], List[Tuple[bytes, T]]]]:
if self.trivial:
return iterator
return self.filter(iterator)
def has_ignore(dirpath: bytes) -> bool:
return os.path.exists(os.path.join(dirpath, b".gitignore"))
|
