1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
# Copyright (C) 2017 Petr Dlouhý
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import unittest
from collections import namedtuple
import linkcheck.configuration
from linkcheck.cache.results import ResultCache
from linkcheck.cache.urlqueue import Empty, NUM_PUTS_CLEANUP, UrlQueue
UrlData = namedtuple("UrlData", "url cache_url aggregate has_result")
Aggregate = namedtuple("Aggregate", "result_cache")
class TestUrlQueue(unittest.TestCase):
def setUp(self):
config = linkcheck.configuration.Configuration()
self.result_cache = ResultCache(config["resultcachesize"])
self.urlqueue = UrlQueue()
self.urldata1 = UrlData(
url="Foo",
cache_url="Foo",
aggregate=Aggregate(result_cache=self.result_cache),
has_result=True,
)
def test_max_allowed_urls_bad_value(self):
with self.assertRaises(ValueError):
UrlQueue(max_allowed_urls=0)
with self.assertRaises(ValueError):
UrlQueue(max_allowed_urls=-1)
def test_qsize(self):
""" Test qsize() """
self.assertEqual(self.urlqueue.qsize(), 0)
self.urlqueue.put(self.urldata1)
self.assertEqual(self.urlqueue.qsize(), 1)
def test_empty(self):
""" Test empty() """
self.assertEqual(self.urlqueue.empty(), True)
self.urlqueue.put(self.urldata1)
self.assertEqual(self.urlqueue.empty(), False)
def test_get_empty(self):
""" Test, that get() with empty queue throws Empty """
with self.assertRaises(Empty):
self.assertEqual(self.urlqueue.get(0), None)
def test_get_negative_timeout(self):
""" Test, that get() with negative timeout throws ValueError """
with self.assertRaises(ValueError):
self.assertEqual(self.urlqueue.get(-1), None)
def test_put_get(self):
"""
Test, that after put() we can get()
the item and it can be get only once
"""
self.urlqueue.put(self.urldata1)
cached_item = self.result_cache.get_result(self.urldata1)
self.assertEqual(cached_item, None)
self.assertEqual(self.urlqueue.get(), self.urldata1)
with self.assertRaises(Empty):
self.assertEqual(self.urlqueue.get(0), None)
def test_put_has_result_false(self):
"""
Test, that element with has_result=False
is put() on the end of queue
"""
self.urlqueue.put(self.urldata1)
urldata = UrlData(
url="Bar",
cache_url="Bar",
aggregate=Aggregate(result_cache=self.result_cache),
has_result=False,
)
self.urlqueue.put(urldata)
self.assertEqual(self.urlqueue.get(), self.urldata1)
self.assertEqual(self.urlqueue.get(), urldata)
with self.assertRaises(Empty):
self.assertEqual(self.urlqueue.get(0), None)
def test_put_has_result_true(self):
"""
Test, that element with has_result=True
is put() on the beginning of queue
"""
self.urlqueue.put(self.urldata1)
urldata = UrlData(
url="Bar",
cache_url="Bar",
aggregate=Aggregate(result_cache=self.result_cache),
has_result=True,
)
self.urlqueue.put(urldata)
self.assertEqual(self.urlqueue.get(), urldata)
self.assertEqual(self.urlqueue.get(), self.urldata1)
with self.assertRaises(Empty):
self.assertEqual(self.urlqueue.get(0), None)
def test_put_cache(self):
"""
Test, that making put() on two elements with same
cache_url adds only one element
"""
self.urlqueue.put(self.urldata1)
urldata = UrlData(
url="Bar",
cache_url="Foo",
aggregate=Aggregate(result_cache=self.result_cache),
has_result=True,
)
self.urlqueue.put(urldata)
self.assertEqual(self.urlqueue.qsize(), 1)
self.assertEqual(self.urlqueue.get(), self.urldata1)
with self.assertRaises(Empty):
self.assertEqual(self.urlqueue.get(0), None)
def test_cleanup(self):
"""
Test, that after adding NUM_PUTS_CLEANUP elements
the queue is cleaned up.
Whether the cleanup is was performed is determined,
that element in cache is now on top of the queue.
"""
for i in range(NUM_PUTS_CLEANUP - 1):
self.urlqueue.put(
UrlData(
url="Bar",
cache_url="Bar address %s" % i,
aggregate=Aggregate(result_cache=self.result_cache),
has_result=False,
),
)
self.assertEqual(self.urlqueue.qsize(), NUM_PUTS_CLEANUP - 1)
urldata = UrlData(
url="Bar",
cache_url="Bar address",
aggregate=Aggregate(result_cache=self.result_cache),
has_result=False,
)
self.result_cache.add_result("Bar address 2", "asdf")
self.urlqueue.put(urldata)
self.assertEqual(self.urlqueue.qsize(), NUM_PUTS_CLEANUP)
self.assertEqual(self.urlqueue.get().cache_url, "Bar address 2")
|
