1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
|
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
# Author: Mauro Soria
import re
import threading
from lib.utils.fmt import safequote, uniq, lowercase, uppercase, capitalize
from lib.utils.file import File, FileUtils
class Dictionary(object):
def __init__(
self,
paths,
extensions,
suffixes=None,
prefixes=None,
lowercase=False,
uppercase=False,
capitalization=False,
force_extensions=False,
exclude_extensions=[],
no_extension=False,
only_selected=False,
):
self.entries = []
self.current_index = 0
self.condition = threading.Lock()
self._extensions = extensions
self._exclude_extensions = exclude_extensions
self._prefixes = prefixes
self._suffixes = suffixes
self._paths = paths
self._force_extensions = force_extensions
self._no_extension = no_extension
self._only_selected = only_selected
self.lowercase = lowercase
self.uppercase = uppercase
self.capitalization = capitalization
self.dictionary_files = [File(path) for path in self.paths]
self.generate()
@property
def extensions(self):
return self._extensions
@extensions.setter
def extensions(self, value):
self._extensions = value
@property
def paths(self):
return self._paths
@paths.setter
def paths(self, paths):
self._paths = paths
"""
Dictionary.generate() behaviour
Classic dirsearch wordlist:
1. If %EXT% keyword is present, append one with each extension REPLACED.
2. If the special word is no present, append line unmodified.
Forced extensions wordlist (NEW):
This type of wordlist processing is a mix between classic processing
and DirBuster processing.
1. If %EXT% keyword is present in the line, immediately process as "classic dirsearch" (1).
2. If the line does not include the special word AND is NOT terminated by a slash,
append one with each extension APPENDED (line.ext) and ONLYE ONE with a slash.
3. If the line does not include the special word and IS ALREADY terminated by slash,
append line unmodified.
"""
def generate(self):
reext = re.compile(r"\%ext\%", re.IGNORECASE).sub
result = []
# Enable to use multiple dictionaries at once
for dict_file in self.dictionary_files:
for line in uniq(dict_file.get_lines(), filt=True):
# Skip comments
if line.startswith("#"):
continue
if line.startswith("/"):
line = line[1:]
if self._no_extension:
line = line[0] + line[1:].split(".")[0]
# Skip dummy paths
if line == ".":
continue
# Skip if the path contains excluded extensions
if self._exclude_extensions and (
any(["." + extension in line for extension in self._exclude_extensions])
):
continue
# Classic dirsearch wordlist processing (with %EXT% keyword)
if "%ext%" in line.lower():
for extension in self._extensions:
newline = reext(extension, line)
result.append(newline)
# If forced extensions is used and the path is not a directory ... (terminated by /)
# process line like a forced extension.
elif self._force_extensions and not line.rstrip().endswith("/") and "." not in line:
for extension in self._extensions:
result.append(line + "." + extension)
result.append(line)
result.append(line + "/")
# Append line unmodified.
else:
if not self._only_selected or any(
[line.endswith("." + extension) for extension in self.extensions]
):
result.append(line)
# Some custom changes
for entry in uniq(result):
entries = [entry]
for pref in self._prefixes:
if not entry.startswith(pref):
entries.append(pref + entry)
for suff in self._suffixes:
if not entry.endswith("/") and not entry.endswith(suff):
entries.append(entry + suff)
if self.lowercase:
self.entries.extend(lowercase(entries))
elif self.uppercase:
self.entries.extend(uppercase(entries))
elif self.capitalization:
self.entries.extend(capitalize(entries))
else:
self.entries.extend(entries)
del result
# Get ignore paths for status codes.
# More information: https://github.com/maurosoria/dirsearch#Blacklist
@staticmethod
def generate_blacklists(extensions, script_path):
reext = re.compile(r"\%ext\%", re.IGNORECASE).sub
blacklists = {}
for status in [400, 403, 500]:
blacklist_file_name = FileUtils.build_path(script_path, "db")
blacklist_file_name = FileUtils.build_path(
blacklist_file_name, "{}_blacklist.txt".format(status)
)
if not FileUtils.can_read(blacklist_file_name):
# Skip if cannot read file
continue
blacklists[status] = []
for line in FileUtils.get_lines(blacklist_file_name):
# Skip comments
if line.lstrip().startswith("#"):
continue
if line.startswith("/"):
line = line[1:]
# Classic dirsearch blacklist processing (with %EXT% keyword)
if "%ext%" in line.lower():
for extension in extensions:
entry = reext.sub(extension, line)
blacklists[status].append(entry)
# Forced extensions is not used here because -r is only used for wordlist,
# applying in blacklist may create false negatives
else:
blacklists[status].append(line)
return blacklists
def next_with_index(self, base_path=None):
self.condition.acquire()
try:
result = self.entries[self.current_index]
except IndexError:
self.condition.release()
raise StopIteration
self.current_index = self.current_index + 1
current_index = self.current_index
self.condition.release()
return current_index, result
def __next__(self, base_path=None):
_, path = self.next_with_index(base_path)
return safequote(path)
def reset(self):
self.condition.acquire()
self.current_index = 0
self.condition.release()
def __len__(self):
return len(self.entries)
|
