1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
from collections import Counter, OrderedDict
from typing import Dict, Iterable, List, Optional
from torchtext._torchtext import Vocab as VocabPybind
from .vocab import Vocab
def vocab(
ordered_dict: Dict, min_freq: int = 1, specials: Optional[List[str]] = None, special_first: bool = True
) -> Vocab:
r"""Factory method for creating a vocab object which maps tokens to indices.
Note that the ordering in which key value pairs were inserted in the `ordered_dict` will be respected when building the vocab.
Therefore if sorting by token frequency is important to the user, the `ordered_dict` should be created in a way to reflect this.
Args:
ordered_dict: Ordered Dictionary mapping tokens to their corresponding occurance frequencies.
min_freq: The minimum frequency needed to include a token in the vocabulary.
specials: Special symbols to add. The order of supplied tokens will be preserved.
special_first: Indicates whether to insert symbols at the beginning or at the end.
Returns:
torchtext.vocab.Vocab: A `Vocab` object
Examples:
>>> from torchtext.vocab import vocab
>>> from collections import Counter, OrderedDict
>>> counter = Counter(["a", "a", "b", "b", "b"])
>>> sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: x[1], reverse=True)
>>> ordered_dict = OrderedDict(sorted_by_freq_tuples)
>>> v1 = vocab(ordered_dict)
>>> print(v1['a']) #prints 1
>>> print(v1['out of vocab']) #raise RuntimeError since default index is not set
>>> tokens = ['e', 'd', 'c', 'b', 'a']
>>> #adding <unk> token and default index
>>> unk_token = '<unk>'
>>> default_index = -1
>>> v2 = vocab(OrderedDict([(token, 1) for token in tokens]), specials=[unk_token])
>>> v2.set_default_index(default_index)
>>> print(v2['<unk>']) #prints 0
>>> print(v2['out of vocab']) #prints -1
>>> #make default index same as index of unk_token
>>> v2.set_default_index(v2[unk_token])
>>> v2['out of vocab'] is v2[unk_token] #prints True
"""
specials = specials or []
for token in specials:
ordered_dict.pop(token, None)
tokens = []
# Save room for special tokens
for token, freq in ordered_dict.items():
if freq >= min_freq:
tokens.append(token)
if special_first:
tokens[0:0] = specials
else:
tokens.extend(specials)
return Vocab(VocabPybind(tokens, None))
def build_vocab_from_iterator(
iterator: Iterable,
min_freq: int = 1,
specials: Optional[List[str]] = None,
special_first: bool = True,
max_tokens: Optional[int] = None,
) -> Vocab:
"""
Build a Vocab from an iterator.
Args:
iterator: Iterator used to build Vocab. Must yield list or iterator of tokens.
min_freq: The minimum frequency needed to include a token in the vocabulary.
specials: Special symbols to add. The order of supplied tokens will be preserved.
special_first: Indicates whether to insert symbols at the beginning or at the end.
max_tokens: If provided, creates the vocab from the `max_tokens - len(specials)` most frequent tokens.
Returns:
torchtext.vocab.Vocab: A `Vocab` object
Examples:
>>> #generating vocab from text file
>>> import io
>>> from torchtext.vocab import build_vocab_from_iterator
>>> def yield_tokens(file_path):
>>> with io.open(file_path, encoding = 'utf-8') as f:
>>> for line in f:
>>> yield line.strip().split()
>>> vocab = build_vocab_from_iterator(yield_tokens(file_path), specials=["<unk>"])
"""
counter = Counter()
for tokens in iterator:
counter.update(tokens)
specials = specials or []
# First sort by descending frequency, then lexicographically
sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
if max_tokens is None:
ordered_dict = OrderedDict(sorted_by_freq_tuples)
else:
assert len(specials) < max_tokens, "len(specials) >= max_tokens, so the vocab will be entirely special tokens."
ordered_dict = OrderedDict(sorted_by_freq_tuples[: max_tokens - len(specials)])
word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials, special_first=special_first)
return word_vocab
|
