1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
|
#' Basic tokenizers
#'
#' These functions perform basic tokenization into words, sentences, paragraphs,
#' lines, and characters. The functions can be piped into one another to create
#' at most two levels of tokenization. For instance, one might split a text into
#' paragraphs and then word tokens, or into sentences and then word tokens.
#'
#' @name basic-tokenizers
#' @param x A character vector or a list of character vectors to be tokenized.
#' If \code{x} is a character vector, it can be of any length, and each element
#' will be tokenized separately. If \code{x} is a list of character vectors,
#' where each element of the list should have a length of 1.
#' @param lowercase Should the tokens be made lower case? The default value
#' varies by tokenizer; it is only \code{TRUE} by default for the tokenizers
#' that you are likely to use last.
#' @param strip_non_alphanum Should punctuation and white space be stripped?
#' @param strip_punct Should punctuation be stripped?
#' @param strip_numeric Should numbers be stripped?
#' @param paragraph_break A string identifying the boundary between two
#' paragraphs.
#' @param stopwords A character vector of stop words to be excluded.
#' @param pattern A regular expression that defines the split.
#' @param simplify \code{FALSE} by default so that a consistent value is
#' returned regardless of length of input. If \code{TRUE}, then an input with
#' a single element will return a character vector of tokens instead of a
#' list.
#' @return A list of character vectors containing the tokens, with one element
#' in the list for each element that was passed as input. If \code{simplify =
#' TRUE} and only a single element was passed as input, then the output is a
#' character vector of tokens.
#' @importFrom stringi stri_split_boundaries stri_trans_tolower stri_trim_both
#' stri_replace_all_charclass stri_split_fixed stri_split_lines
#' stri_split_regex stri_subset_charclass
#' @examples
#' song <- paste0("How many roads must a man walk down\n",
#' "Before you call him a man?\n",
#' "How many seas must a white dove sail\n",
#' "Before she sleeps in the sand?\n",
#' "\n",
#' "How many times must the cannonballs fly\n",
#' "Before they're forever banned?\n",
#' "The answer, my friend, is blowin' in the wind.\n",
#' "The answer is blowin' in the wind.\n")
#'
#' tokenize_words(song)
#' tokenize_words(song, strip_punct = FALSE)
#' tokenize_sentences(song)
#' tokenize_paragraphs(song)
#' tokenize_lines(song)
#' tokenize_characters(song)
NULL
#' @export
#' @rdname basic-tokenizers
tokenize_characters <-
function(x,
lowercase = TRUE,
strip_non_alphanum = TRUE,
simplify = FALSE) {
UseMethod("tokenize_characters")
}
#' @export
tokenize_characters.data.frame <- function(x,
lowercase = TRUE,
strip_non_alphanum = TRUE,
simplify = FALSE) {
x <- corpus_df_as_corpus_vector(x)
tokenize_characters(x, lowercase, strip_non_alphanum, simplify)
}
#' @export
tokenize_characters.default <- function(x,
lowercase = TRUE,
strip_non_alphanum = TRUE,
simplify = FALSE) {
check_input(x)
named <- names(x)
if (lowercase)
x <- stri_trans_tolower(x)
if (strip_non_alphanum)
x <-
stri_replace_all_charclass(x, "[[:punct:][:whitespace:]]", "")
out <- stri_split_boundaries(x, type = "character")
if (!is.null(named))
names(out) <- named
simplify_list(out, simplify)
}
#' @export
#' @rdname basic-tokenizers
tokenize_words <- function(x, lowercase = TRUE, stopwords = NULL,
strip_punct = TRUE, strip_numeric = FALSE,
simplify = FALSE) {
UseMethod("tokenize_words")
}
#' @export
tokenize_words.data.frame <- function(x,
lowercase = TRUE,
stopwords = NULL,
strip_punct = TRUE,
strip_numeric = FALSE,
simplify = FALSE) {
x <- corpus_df_as_corpus_vector(x)
tokenize_words(x, lowercase, stopwords, strip_punct, strip_numeric, simplify)
}
#' @export
tokenize_words.default <- function(x, lowercase = TRUE, stopwords = NULL,
strip_punct = TRUE, strip_numeric = FALSE,
simplify = FALSE) {
check_input(x)
named <- names(x)
if (lowercase) x <- stri_trans_tolower(x)
out <- stri_split_boundaries(x, type = "word",
skip_word_none = strip_punct,
skip_word_number = strip_numeric)
if (!strip_punct) {
out <- lapply(out, stri_subset_charclass, "\\p{WHITESPACE}", negate = TRUE)
}
if (!is.null(named)) names(out) <- named
if (!is.null(stopwords)) out <- lapply(out, remove_stopwords, stopwords)
simplify_list(out, simplify)
}
#' @export
#' @rdname basic-tokenizers
tokenize_sentences <-
function(x,
lowercase = FALSE,
strip_punct = FALSE,
simplify = FALSE) {
UseMethod("tokenize_sentences")
}
#' @export
tokenize_sentences.data.frame <-
function(x,
lowercase = FALSE,
strip_punct = FALSE,
simplify = FALSE) {
x <- corpus_df_as_corpus_vector(x)
tokenize_sentences(x, lowercase, strip_punct, simplify)
}
#' @export
tokenize_sentences.default <-
function(x,
lowercase = FALSE,
strip_punct = FALSE,
simplify = FALSE) {
check_input(x)
named <- names(x)
x <- stri_replace_all_charclass(x, "[[:whitespace:]]", " ")
out <-
stri_split_boundaries(x, type = "sentence", skip_word_none = FALSE)
out <- lapply(out, stri_trim_both)
if (lowercase)
out <- lapply(out, stri_trans_tolower)
if (strip_punct)
out <-
lapply(out, stri_replace_all_charclass, "[[:punct:]]", "")
if (!is.null(named))
names(out) <- named
simplify_list(out, simplify)
}
#' @export
#' @rdname basic-tokenizers
tokenize_lines <- function(x, simplify = FALSE) {
UseMethod("tokenize_lines")
}
#' @export
tokenize_lines.data.frame <- function(x, simplify = FALSE) {
x <- corpus_df_as_corpus_vector(x)
tokenize_lines(x, simplify)
}
#' @export
tokenize_lines.default <- function(x, simplify = FALSE) {
check_input(x)
named <- names(x)
out <- stri_split_lines(x, omit_empty = TRUE)
if (!is.null(named))
names(out) <- named
simplify_list(out, simplify)
}
#' @export
#' @rdname basic-tokenizers
tokenize_paragraphs <-
function(x,
paragraph_break = "\n\n",
simplify = FALSE) {
UseMethod("tokenize_paragraphs")
}
#' @export
tokenize_paragraphs.data.frame <-
function(x,
paragraph_break = "\n\n",
simplify = FALSE) {
x <- corpus_df_as_corpus_vector(x)
tokenize_paragraphs(x, paragraph_break, simplify)
}
#' @export
tokenize_paragraphs.default <-
function(x,
paragraph_break = "\n\n",
simplify = FALSE) {
check_input(x)
named <- names(x)
out <-
stri_split_fixed(x, pattern = paragraph_break, omit_empty = TRUE)
out <-
lapply(out, stri_replace_all_charclass, "[[:whitespace:]]", " ")
if (!is.null(named))
names(out) <- named
simplify_list(out, simplify)
}
#' @export
#' @rdname basic-tokenizers
tokenize_regex <- function(x,
pattern = "\\s+",
simplify = FALSE) {
UseMethod("tokenize_regex")
}
#' @export
tokenize_regex.data.frame <-
function(x,
pattern = "\\s+",
simplify = FALSE) {
x <- corpus_df_as_corpus_vector(x)
tokenize_regex(x, pattern, simplify)
}
#' @export
tokenize_regex.default <-
function(x,
pattern = "\\s+",
simplify = FALSE) {
check_input(x)
named <- names(x)
out <- stri_split_regex(x, pattern = pattern, omit_empty = TRUE)
if (!is.null(named))
names(out) <- named
simplify_list(out, simplify)
}
|
