1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
# -*- coding: utf-8 -*-
"""
test_search
~~~~~~~~~~~
Test the search index builder.
:copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from collections import namedtuple
from six import BytesIO
from docutils import frontend, utils
from docutils.parsers import rst
from sphinx.search import IndexBuilder
from sphinx.util import jsdump
import pytest
DummyEnvironment = namedtuple('DummyEnvironment', ['version', 'domains'])
class DummyDomain(object):
def __init__(self, data):
self.data = data
self.object_types = {}
def get_objects(self):
return self.data
settings = parser = None
def setup_module():
global settings, parser
optparser = frontend.OptionParser(components=(rst.Parser,))
settings = optparser.get_default_values()
parser = rst.Parser()
def jsload(path):
searchindex = path.text()
assert searchindex.startswith('Search.setIndex(')
return jsdump.loads(searchindex[16:-2])
def is_registered_term(index, keyword):
return index['terms'].get(keyword, []) != []
FILE_CONTENTS = '''\
section_title
=============
.. test that comments are not indexed: boson
test that non-comments are indexed: fermion
'''
@pytest.mark.sphinx(testroot='ext-viewcode')
def test_objects_are_escaped(app, status, warning):
app.builder.build_all()
searchindex = (app.outdir / 'searchindex.js').text()
assert searchindex.startswith('Search.setIndex(')
index = jsdump.loads(searchindex[16:-2])
assert 'n::Array<T, d>' in index.get('objects').get('') # n::Array<T,d> is escaped
@pytest.mark.sphinx(testroot='search')
def test_meta_keys_are_handled_for_language_en(app, status, warning):
app.builder.build_all()
searchindex = jsload(app.outdir / 'searchindex.js')
assert not is_registered_term(searchindex, 'thisnoteith')
assert is_registered_term(searchindex, 'thisonetoo')
assert is_registered_term(searchindex, 'findthiskei')
assert is_registered_term(searchindex, 'thistoo')
assert not is_registered_term(searchindex, 'onlygerman')
assert is_registered_term(searchindex, 'notgerman')
assert not is_registered_term(searchindex, 'onlytoogerman')
@pytest.mark.skip('Search languages using snowball are currently disabled')
@pytest.mark.sphinx(testroot='search', confoverrides={'html_search_language': 'de'})
def test_meta_keys_are_handled_for_language_de(app, status, warning):
app.builder.build_all()
searchindex = jsload(app.outdir / 'searchindex.js')
assert not is_registered_term(searchindex, 'thisnoteith')
assert is_registered_term(searchindex, 'thisonetoo')
assert not is_registered_term(searchindex, 'findthiskei')
assert not is_registered_term(searchindex, 'thistoo')
assert is_registered_term(searchindex, 'onlygerman')
assert not is_registered_term(searchindex, 'notgerman')
assert is_registered_term(searchindex, 'onlytoogerman')
@pytest.mark.sphinx(testroot='search')
def test_stemmer_does_not_remove_short_words(app, status, warning):
app.builder.build_all()
searchindex = (app.outdir / 'searchindex.js').text()
assert 'zfs' in searchindex
@pytest.mark.sphinx(testroot='search')
def test_stemmer(app, status, warning):
searchindex = jsload(app.outdir / 'searchindex.js')
print(searchindex)
assert is_registered_term(searchindex, 'findthisstemmedkei')
assert is_registered_term(searchindex, 'intern')
@pytest.mark.sphinx(testroot='search')
def test_term_in_heading_and_section(app, status, warning):
searchindex = (app.outdir / 'searchindex.js').text()
# if search term is in the title of one doc and in the text of another
# both documents should be a hit in the search index as a title,
# respectively text hit
assert 'textinhead:1' in searchindex
assert 'textinhead:0' in searchindex
@pytest.mark.sphinx(testroot='search')
def test_term_in_raw_directive(app, status, warning):
searchindex = jsload(app.outdir / 'searchindex.js')
assert not is_registered_term(searchindex, 'raw')
assert is_registered_term(searchindex, 'rawword')
assert not is_registered_term(searchindex, 'latex_keyword')
def test_IndexBuilder():
domain = DummyDomain([('objname', 'objdispname', 'objtype', 'docname', '#anchor', 1),
('objname2', 'objdispname2', 'objtype2', 'docname2', '', -1)])
env = DummyEnvironment('1.0', {'dummy': domain})
doc = utils.new_document(b'test data', settings)
doc['file'] = 'dummy'
parser.parse(FILE_CONTENTS, doc)
# feed
index = IndexBuilder(env, 'en', {}, None)
index.feed('docname', 'filename', 'title', doc)
index.feed('docname2', 'filename2', 'title2', doc)
assert index._titles == {'docname': 'title', 'docname2': 'title2'}
assert index._filenames == {'docname': 'filename', 'docname2': 'filename2'}
assert index._mapping == {
'fermion': {'docname', 'docname2'},
'comment': {'docname', 'docname2'},
'non': {'docname', 'docname2'},
'index': {'docname', 'docname2'},
'test': {'docname', 'docname2'}
}
assert index._title_mapping == {'section_titl': {'docname', 'docname2'}}
assert index._objtypes == {}
assert index._objnames == {}
# freeze
assert index.freeze() == {
'docnames': ('docname', 'docname2'),
'envversion': '1.0',
'filenames': ['filename', 'filename2'],
'objects': {'': {'objname': (0, 0, 1, '#anchor')}},
'objnames': {0: ('dummy', 'objtype', 'objtype')},
'objtypes': {0: 'dummy:objtype'},
'terms': {'comment': [0, 1],
'fermion': [0, 1],
'index': [0, 1],
'non': [0, 1],
'test': [0, 1]},
'titles': ('title', 'title2'),
'titleterms': {'section_titl': [0, 1]}
}
assert index._objtypes == {('dummy', 'objtype'): 0}
assert index._objnames == {0: ('dummy', 'objtype', 'objtype')}
# dump / load
stream = BytesIO()
index.dump(stream, 'pickle')
stream.seek(0)
index2 = IndexBuilder(env, 'en', {}, None)
index2.load(stream, 'pickle')
assert index2._titles == index._titles
assert index2._filenames == index._filenames
assert index2._mapping == index._mapping
assert index2._title_mapping == index._title_mapping
assert index2._objtypes == {}
assert index2._objnames == {}
# freeze after load
assert index2.freeze() == index.freeze()
assert index2._objtypes == index._objtypes
assert index2._objnames == index._objnames
# prune
index.prune(['docname2'])
assert index._titles == {'docname2': 'title2'}
assert index._filenames == {'docname2': 'filename2'}
assert index._mapping == {
'fermion': {'docname2'},
'comment': {'docname2'},
'non': {'docname2'},
'index': {'docname2'},
'test': {'docname2'}
}
assert index._title_mapping == {'section_titl': {'docname2'}}
assert index._objtypes == {('dummy', 'objtype'): 0}
assert index._objnames == {0: ('dummy', 'objtype', 'objtype')}
# freeze after prune
assert index.freeze() == {
'docnames': ('docname2',),
'envversion': '1.0',
'filenames': ['filename2'],
'objects': {},
'objnames': {0: ('dummy', 'objtype', 'objtype')},
'objtypes': {0: 'dummy:objtype'},
'terms': {'comment': 0,
'fermion': 0,
'index': 0,
'non': 0,
'test': 0},
'titles': ('title2',),
'titleterms': {'section_titl': 0}
}
assert index._objtypes == {('dummy', 'objtype'): 0}
assert index._objnames == {0: ('dummy', 'objtype', 'objtype')}
|
