1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import logging
from lxml import etree
import re
from markupsafe import Markup
from odoo.addons.website.controllers.main import Website
from odoo.addons.website.tools import distance, MockRequest
import odoo.tests
from odoo.tests.common import TransactionCase
_logger = logging.getLogger(__name__)
@odoo.tests.tagged('-at_install', 'post_install')
class TestFuzzy(TransactionCase):
def test_01_fuzzy_names(self):
# Models from other modules commented out on commit: they make the test much longer
# Last results observed across modules:
# - 698 words in target dictionary
# - 21 wrong guesses over 9379 tested typos (0.22%)
# - Duration: ~5.7 seconds
fields_per_model = {
'website.page': ['name', 'arch'],
# 'product.public.category': ['name', 'website_description'],
# 'product.template': ['name', 'description_sale'],
# 'blog.blog': ['name', 'subtitle'],
# 'blog.post': ['name', 'subtitle'],
# 'slide.channel': ['name', 'description_short'],
# 'slide.slide': ['name', 'description'],
# 'event.event': ['name', 'subtitle'],
}
match_pattern = '\\w{4,}'
words = set()
for model_name, fields in fields_per_model.items():
if model_name not in self.env:
continue
model = self.env[model_name]
if 'description' not in fields and 'description' in model:
fields.append('description') # larger target dataset
records = model.sudo().search_read([], fields, limit=100)
for record in records:
for field, value in record.items():
if isinstance(value, str):
if field == 'arch':
view_arch = etree.fromstring(value.encode('utf-8'))
value = ' '.join(view_arch.itertext())
for word in re.findall(match_pattern, value):
words.add(word.lower())
_logger.info("%s words in target dictionary", len(words))
website = self.env.ref('website.default_website')
typos = {}
def add_typo(expected, typo):
if typo not in words:
typos.setdefault(typo, set()).add(expected)
for search in words:
for index in range(2, len(search)):
# swap letters
if search[index] != search[index - 1]:
add_typo(search, search[:index - 1] + search[index] + search[index - 1] + search[index + 1:])
# miss letter
if len(search) > 4:
add_typo(search, search[:index - 1] + search[index:])
# wrong letter
add_typo(search, search[:index - 1] + '!' + search[index:])
words = list(words)
words.sort() # guarantee results stability
mismatch_count = 0
for search, expected in typos.items():
fuzzy_guess = website._search_find_fuzzy_term({}, search, word_list=words)
if not fuzzy_guess or (fuzzy_guess not in expected and fuzzy_guess not in [exp[:-1] for exp in expected]):
mismatch_count += 1
_logger.info("'%s' fuzzy matched to '%s' instead of %s", search, fuzzy_guess, expected)
ratio = 100.0 * mismatch_count / len(typos)
_logger.info("%s wrong guesses over %s tested typos (%.2f%%)", mismatch_count, len(typos), ratio)
typos.clear()
words.clear()
self.assertTrue(ratio < 1, "Too many wrong fuzzy guesses")
def test_02_distance(self):
self.assertEqual(distance("gravity", "granity", 3), 1)
self.assertEqual(distance("gravity", "graity", 3), 1)
self.assertEqual(distance("gravity", "grait", 3), 2)
self.assertEqual(distance("gravity", "griaty", 3), 3)
self.assertEqual(distance("gravity", "giraty", 3), 3)
self.assertEqual(distance("gravity", "giraty", 2), -1)
self.assertEqual(distance("gravity", "girafe", 3), -1)
self.assertEqual(distance("warranty", "warantl", 3), 2)
# non-optimized cases still have to return correct results
self.assertEqual(distance("warranty", "warranty", 3), 0)
self.assertEqual(distance("", "warranty", 3), -1)
self.assertEqual(distance("", "warranty", 10), 8)
self.assertEqual(distance("warranty", "", 10), 8)
self.assertEqual(distance("", "", 10), 0)
@odoo.tests.tagged('-at_install', 'post_install')
class TestAutoComplete(TransactionCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.website = cls.env['website'].browse(1)
cls.WebsiteController = Website()
cls.options = {
'displayDescription': True,
}
cls.expectedParts = {
'name': True,
'description': True,
'website_url': True,
}
texts = [
"This page only matches few",
"This page matches both few and many",
"This page only matches many",
"Many results contain this page",
"How many times does the word many appear",
"Will there be many results",
"Welcome to our many friends next week-end",
"This should only be approximately matched",
]
for text in texts:
cls._create_page(text, text, f"/{text.lower().replace(' ', '-')}")
@classmethod
def _create_page(cls, name, content, url):
cls.env['website.page'].create({
'name': name,
'type': 'qweb',
'arch': f'<div>{content}</div>',
'url': url,
'is_published': True,
})
def _autocomplete(self, term):
""" Calls the autocomplete for a given term and performs general checks """
with MockRequest(self.env, website=self.website):
suggestions = self.WebsiteController.autocomplete(
search_type="pages", term=term, max_nb_chars=50, options=self.options,
)
if suggestions['results_count']:
self.assertDictEqual(self.expectedParts, suggestions['parts'],
f"Parts should contain {self.expectedParts.keys()}")
for result in suggestions['results']:
self.assertEqual("fa-file-o", result['_fa'], "Expect an fa icon")
for field in suggestions['parts'].keys():
value = result[field]
if value:
self.assertTrue(
isinstance(value, Markup),
f"All fields should be wrapped in Markup: found {type(value)}: '{value}' in {field}"
)
return suggestions
def _check_highlight(self, term, value):
""" Verifies if a term is highlighted in a value """
self.assertTrue(f'<span class="text-primary">{term}</span>' in value.lower(),
"Term must be highlighted")
def test_01_few_results(self):
""" Tests an autocomplete with exact match and less than the maximum number of results """
suggestions = self._autocomplete("few")
self.assertEqual(2, suggestions['results_count'], "Text data contains two pages with 'few'")
self.assertEqual(2, len(suggestions['results']), "All results must be present")
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
for result in suggestions['results']:
self._check_highlight("few", result['name'])
self._check_highlight("few", result['description'])
def test_02_many_results(self):
""" Tests an autocomplete with exact match and more than the maximum number of results """
suggestions = self._autocomplete("many")
self.assertEqual(6, suggestions['results_count'], "Test data contains six pages with 'many'")
self.assertEqual(5, len(suggestions['results']), "Results must be limited to 5")
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
for result in suggestions['results']:
self._check_highlight("many", result['name'])
self._check_highlight("many", result['description'])
def test_03_no_result(self):
""" Tests an autocomplete without matching results """
suggestions = self._autocomplete("nothing")
self.assertEqual(0, suggestions['results_count'], "Text data contains no page with 'nothing'")
self.assertEqual(0, len(suggestions['results']), "No result must be present")
def test_04_fuzzy_results(self):
""" Tests an autocomplete with fuzzy matching results """
suggestions = self._autocomplete("appoximtly")
self.assertEqual("approximately", suggestions['fuzzy_search'], "")
self.assertEqual(1, suggestions['results_count'], "Text data contains one page with 'approximately'")
self.assertEqual(1, len(suggestions['results']), "Single result must be present")
for result in suggestions['results']:
self._check_highlight("approximately", result['name'])
self._check_highlight("approximately", result['description'])
def test_05_long_url(self):
""" Ensures that long URL do not get truncated """
url = "/this-url-is-so-long-it-would-be-truncated-without-the-fix"
self._create_page("Too long", "Way too long URL", url)
suggestions = self._autocomplete("long url")
self.assertEqual(1, suggestions['results_count'], "Text data contains one page with 'long url'")
self.assertEqual(1, len(suggestions['results']), "Single result must be present")
self.assertEqual(url, suggestions['results'][0]['website_url'], 'URL must not be truncated')
def test_06_case_insensitive_results(self):
""" Tests an autocomplete with exact match and more than the maximum
number of results.
"""
suggestions = self._autocomplete("Many")
self.assertEqual(6, suggestions['results_count'], "Test data contains six pages with 'Many'")
self.assertEqual(5, len(suggestions['results']), "Results must be limited to 5")
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
for result in suggestions['results']:
self._check_highlight("many", result['name'])
self._check_highlight("many", result['description'])
def test_07_no_fuzzy_for_mostly_number(self):
""" Ensures exact match is used when search contains mostly numbers. """
self._create_page('Product P7935432254U7 page', 'Product P7935432254U7 kangaroo shoes', '/numberpage')
suggestions = self._autocomplete("54321")
self.assertEqual(0, suggestions['results_count'], "Test data contains no exact match")
suggestions = self._autocomplete("54322")
self.assertEqual(1, suggestions['results_count'], "Test data contains one exact match")
suggestions = self._autocomplete("P79355")
self.assertEqual(0, suggestions['results_count'], "Test data contains no exact match")
suggestions = self._autocomplete("P79354")
self.assertEqual(1, suggestions['results_count'], "Test data contains one exact match")
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
suggestions = self._autocomplete("kangroo") # must contain a typo
self.assertEqual(1, suggestions['results_count'], "Test data contains one fuzzy match")
self.assertTrue(suggestions['fuzzy_search'], "Expects a fuzzy match")
def test_08_fuzzy_classic_numbers(self):
""" Ensures fuzzy match is used when search contains a few numbers. """
self._create_page('iPhone 6', 'iPhone6', '/iphone6')
suggestions = self._autocomplete("iphone7")
self.assertEqual(1, suggestions['results_count'], "Test data contains one fuzzy match")
self.assertTrue(suggestions['fuzzy_search'], "Expects an fuzzy match")
def test_09_hyphen(self):
""" Ensures that hyphen is considered part of word """
suggestions = self._autocomplete("weekend")
self.assertEqual(1, suggestions['results_count'], "Text data contains one page with 'weekend'")
self.assertEqual('week-end', suggestions['fuzzy_search'], "Expects a fuzzy match")
suggestions = self._autocomplete("week-end")
self.assertEqual(1, len(suggestions['results']), "All results must be present")
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
|
