1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
"""This file provides tests to benchmark performance sqlite/file queue
on specific hardware. User can easily evaluate the performance by running this
file directly via `python run_benchmark.py`
"""
from persistqueue import SQLiteQueue
from persistqueue import Queue
import tempfile
import time
BENCHMARK_COUNT = 100
def time_it(func):
def _exec(*args, **kwargs):
start = time.time()
func(*args, **kwargs)
end = time.time()
print(
"\t{} => time used: {:.4f} seconds.".format(
func.__doc__,
(end - start)))
return _exec
class FileQueueBench(object):
"""Benchmark File queue performance."""
def __init__(self, prefix=None):
self.path = prefix
@time_it
def benchmark_file_write(self):
"""Writing <BENCHMARK_COUNT> items."""
self.path = tempfile.mkdtemp('b_file_10000')
q = Queue(self.path)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
assert q.qsize() == BENCHMARK_COUNT
@time_it
def benchmark_file_read_write_false(self):
"""Writing and reading <BENCHMARK_COUNT> items(1 task_done)."""
self.path = tempfile.mkdtemp('b_file_10000')
q = Queue(self.path)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
q.task_done()
assert q.qsize() == 0
@time_it
def benchmark_file_read_write_autosave(self):
"""Writing and reading <BENCHMARK_COUNT> items(autosave)."""
self.path = tempfile.mkdtemp('b_file_10000')
q = Queue(self.path, autosave=True)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
assert q.qsize() == 0
@time_it
def benchmark_file_read_write_true(self):
"""Writing and reading <BENCHMARK_COUNT> items(many task_done)."""
self.path = tempfile.mkdtemp('b_file_10000')
q = Queue(self.path)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
q.task_done()
assert q.qsize() == 0
@classmethod
def run(cls):
print(cls.__doc__)
ins = cls()
for name in sorted(cls.__dict__):
if name.startswith('benchmark'):
func = getattr(ins, name)
func()
class Sqlite3QueueBench(object):
"""Benchmark Sqlite3 queue performance."""
@time_it
def benchmark_sqlite_write(self):
"""Writing <BENCHMARK_COUNT> items."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
assert q.qsize() == BENCHMARK_COUNT
@time_it
def benchmark_sqlite_read_write_false(self):
"""Writing and reading <BENCHMARK_COUNT> items(1 task_done)."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=False)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
q.task_done()
assert q.qsize() == 0
@time_it
def benchmark_sqlite_read_write_true(self):
"""Writing and reading <BENCHMARK_COUNT> items(many task_done)."""
self.path = tempfile.mkdtemp('b_sql_10000')
q = SQLiteQueue(self.path, auto_commit=True)
for i in range(BENCHMARK_COUNT):
q.put('bench%d' % i)
for i in range(BENCHMARK_COUNT):
q.get()
q.task_done()
assert q.qsize() == 0
@classmethod
def run(cls):
print(cls.__doc__)
ins = cls()
for name in sorted(cls.__dict__):
if name.startswith('benchmark'):
func = getattr(ins, name)
func()
if __name__ == '__main__':
import sys
if len(sys.argv) > 1:
BENCHMARK_COUNT = int(sys.argv[1])
print("<BENCHMARK_COUNT> = {}".format(BENCHMARK_COUNT))
file_bench = FileQueueBench()
file_bench.run()
sql_bench = Sqlite3QueueBench()
sql_bench.run()
|
