1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
from lunr.pipeline import Pipeline
WORDS = {
"a",
"able",
"about",
"across",
"after",
"all",
"almost",
"also",
"am",
"among",
"an",
"and",
"any",
"are",
"as",
"at",
"be",
"because",
"been",
"but",
"by",
"can",
"cannot",
"could",
"dear",
"did",
"do",
"does",
"either",
"else",
"ever",
"every",
"for",
"from",
"get",
"got",
"had",
"has",
"have",
"he",
"her",
"hers",
"him",
"his",
"how",
"however",
"i",
"if",
"in",
"into",
"is",
"it",
"its",
"just",
"least",
"let",
"like",
"likely",
"may",
"me",
"might",
"most",
"must",
"my",
"neither",
"no",
"nor",
"not",
"of",
"off",
"often",
"on",
"only",
"or",
"other",
"our",
"own",
"rather",
"said",
"say",
"says",
"she",
"should",
"since",
"so",
"some",
"than",
"that",
"the",
"their",
"them",
"then",
"there",
"these",
"they",
"this",
"tis",
"to",
"too",
"twas",
"us",
"wants",
"was",
"we",
"were",
"what",
"when",
"where",
"which",
"while",
"who",
"whom",
"why",
"will",
"with",
"would",
"yet",
"you",
"your",
}
def generate_stop_word_filter(stop_words, language=None):
"""Builds a stopWordFilter function from the provided list of stop words.
The built in `stop_word_filter` is built using this factory and can be used
to generate custom `stop_word_filter` for applications or non English
languages.
"""
def stop_word_filter(token, i=None, tokens=None):
if token and str(token) not in stop_words:
return token
# camelCased for for compatibility with lunr.js
label = (
"stopWordFilter-{}".format(language)
if language is not None
else "stopWordFilter"
)
Pipeline.register_function(stop_word_filter, label)
return stop_word_filter
stop_word_filter = generate_stop_word_filter(WORDS)
|
