1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
from multicorn import ForeignDataWrapper
import gc
import sys
from multicorn.compat import unicode_
class MyClass(object):
def __init__(self, num, rand):
self.num = num
self.rand = rand
class GCForeignDataWrapper(ForeignDataWrapper):
def execute(self, quals, columns):
gc.collect()
result = []
for obj in gc.get_objects():
tobj = type(obj)
if isinstance(obj, bytes):
obj = obj.decode('utf8')
elif isinstance(obj, unicode_):
pass
else:
try:
obj = bytes(obj).decode('utf8')
except (UnicodeEncodeError, UnicodeDecodeError):
try:
obj = unicode_(obj)
except (UnicodeEncodeError, UnicodeDecodeError):
obj = unicode_("<NA>")
result.append({'object': obj,
'type': unicode_(tobj),
'id': unicode_(id(obj)),
'refcount': unicode_(sys.getrefcount(obj))})
return result
class MemStressFDW(ForeignDataWrapper):
def __init__(self, options, columns):
self.nb = int(options.get('nb', 100000))
self.options = options
self.columns = columns
super(MemStressFDW, self).__init__(options, columns)
def execute(self, quals, columns):
for i in range(self.nb):
num = i / 100.
yield {'value': str(MyClass(i, num)),
'i': i,
'num': num}
|
