1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
|
## ----setup, include = FALSE---------------------------------------------------
knitr::opts_chunk$set(
collapse = TRUE,
comment = "#>"
)
## -----------------------------------------------------------------------------
# Named list
(corpus_l <- list(man_comes_around = "There's a man goin' 'round takin' names",
wont_back_down = "Well I won't back down, no I won't back down",
bird_on_a_wire = "Like a bird on a wire"))
# Named character vector
(corpus_c <- unlist(corpus_l))
# Data frame
(corpus_d <- data.frame(doc_id = names(corpus_c), text = unname(corpus_c),
stringsAsFactors = FALSE))
## -----------------------------------------------------------------------------
library(tokenizers)
tokens_l <- tokenize_ngrams(corpus_l, n = 2)
tokens_c <- tokenize_ngrams(corpus_c, n = 2)
tokens_d <- tokenize_ngrams(corpus_c, n = 2)
# Are all these identical?
all(identical(tokens_l, tokens_c),
identical(tokens_c, tokens_d),
identical(tokens_l, tokens_d))
## -----------------------------------------------------------------------------
tokens_l
## ---- echo=FALSE--------------------------------------------------------------
sample_tokens_df <- structure(list(doc_id = c("man_comes_around", "man_comes_around",
"man_comes_around", "man_comes_around", "man_comes_around", "man_comes_around",
"wont_back_down", "wont_back_down", "wont_back_down", "wont_back_down",
"wont_back_down", "wont_back_down", "wont_back_down", "wont_back_down",
"wont_back_down", "bird_on_a_wire", "bird_on_a_wire", "bird_on_a_wire",
"bird_on_a_wire", "bird_on_a_wire"), token = c("there's a", "a man",
"man goin", "goin round", "round takin", "takin names", "well i",
"i won't", "won't back", "back down", "down no", "no i", "i won't",
"won't back", "back down", "like a", "a bird", "bird on", "on a",
"a wire")), .Names = c("doc_id", "token"), row.names = c(NA,
-20L), class = "data.frame")
head(sample_tokens_df, 10)
|
