1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
#!/usr/bin/env python
"""
Copyright (c) 2006-2025 sqlmap developers (https://sqlmap.org)
See the file 'LICENSE' for copying permission
"""
try:
import cPickle as pickle
except:
import pickle
import itertools
import os
import sys
import tempfile
import threading
import zlib
from lib.core.compat import xrange
from lib.core.enums import MKSTEMP_PREFIX
from lib.core.exception import SqlmapSystemException
from lib.core.settings import BIGARRAY_CHUNK_SIZE
from lib.core.settings import BIGARRAY_COMPRESS_LEVEL
try:
DEFAULT_SIZE_OF = sys.getsizeof(object())
except TypeError:
DEFAULT_SIZE_OF = 16
def _size_of(instance):
"""
Returns total size of a given instance / object (in bytes)
"""
retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)
if isinstance(instance, dict):
retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
elif hasattr(instance, "__iter__"):
retval += sum(_size_of(_) for _ in instance if _ != instance)
return retval
class Cache(object):
"""
Auxiliary class used for storing cached chunks
"""
def __init__(self, index, data, dirty):
self.index = index
self.data = data
self.dirty = dirty
class BigArray(list):
"""
List-like class used for storing large amounts of data (disk cached)
>>> _ = BigArray(xrange(100000))
>>> _[20] = 0
>>> _[99999]
99999
>>> _ += [0]
>>> _[100000]
0
>>> _ = _ + [1]
>>> _[-1]
1
>>> len([_ for _ in BigArray(xrange(100000))])
100000
"""
def __init__(self, items=None):
self.chunks = [[]]
self.chunk_length = sys.maxsize
self.cache = None
self.filenames = set()
self._lock = threading.Lock()
self._os_remove = os.remove
self._size_counter = 0
for item in (items or []):
self.append(item)
def __add__(self, value):
retval = BigArray(self)
for _ in value:
retval.append(_)
return retval
def __iadd__(self, value):
for _ in value:
self.append(_)
return self
def append(self, value):
with self._lock:
self.chunks[-1].append(value)
if self.chunk_length == sys.maxsize:
self._size_counter += _size_of(value)
if self._size_counter >= BIGARRAY_CHUNK_SIZE:
self.chunk_length = len(self.chunks[-1])
self._size_counter = None
if len(self.chunks[-1]) >= self.chunk_length:
filename = self._dump(self.chunks[-1])
self.chunks[-1] = filename
self.chunks.append([])
def extend(self, value):
for _ in value:
self.append(_)
def pop(self):
if len(self.chunks[-1]) < 1:
self.chunks.pop()
try:
with open(self.chunks[-1], "rb") as f:
self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
except IOError as ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex
raise SqlmapSystemException(errMsg)
return self.chunks[-1].pop()
def index(self, value):
for index in xrange(len(self)):
if self[index] == value:
return index
return ValueError, "%s is not in list" % value
def close(self):
while self.filenames:
filename = self.filenames.pop()
try:
self._os_remove(filename)
except OSError:
pass
def __del__(self):
self.close()
def _dump(self, chunk):
try:
handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
self.filenames.add(filename)
os.close(handle)
with open(filename, "w+b") as f:
f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
return filename
except (OSError, IOError) as ex:
errMsg = "exception occurred while storing data "
errMsg += "to a temporary file ('%s'). Please " % ex
errMsg += "make sure that there is enough disk space left. If problem persists, "
errMsg += "try to set environment variable 'TEMP' to a location "
errMsg += "writeable by the current user"
raise SqlmapSystemException(errMsg)
def _checkcache(self, index):
if (self.cache and self.cache.index != index and self.cache.dirty):
filename = self._dump(self.cache.data)
self.chunks[self.cache.index] = filename
if not (self.cache and self.cache.index == index):
try:
with open(self.chunks[index], "rb") as f:
self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
except Exception as ex:
errMsg = "exception occurred while retrieving data "
errMsg += "from a temporary file ('%s')" % ex
raise SqlmapSystemException(errMsg)
def __getstate__(self):
return self.chunks, self.filenames
def __setstate__(self, state):
self.__init__()
self.chunks, self.filenames = state
def __getitem__(self, y):
length = len(self)
if length == 0:
raise IndexError("BigArray index out of range")
while y < 0:
y += length
index = y // self.chunk_length
offset = y % self.chunk_length
chunk = self.chunks[index]
if isinstance(chunk, list):
return chunk[offset]
else:
self._checkcache(index)
return self.cache.data[offset]
def __setitem__(self, y, value):
index = y // self.chunk_length
offset = y % self.chunk_length
chunk = self.chunks[index]
if isinstance(chunk, list):
chunk[offset] = value
else:
self._checkcache(index)
self.cache.data[offset] = value
self.cache.dirty = True
def __repr__(self):
return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())
def __iter__(self):
for i in xrange(len(self)):
try:
yield self[i]
except IndexError:
break
def __len__(self):
return len(self.chunks[-1]) if len(self.chunks) == 1 else (len(self.chunks) - 1) * self.chunk_length + len(self.chunks[-1])
|
