1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
|
import time
from torchtext.prototype import datasets
def benchmark_construction(name, Dataset):
t0 = time.perf_counter()
print(name, end="")
(d,) = Dataset(data_select=("train",))
print(" construction time {0:.2f}s".format(time.perf_counter() - t0))
del d
def benchmark_raw_construction(name, Dataset):
print(name, end="")
if name in "WMTNewsCrawl":
d = Dataset(data_select=("train",))
else:
d = Dataset()
del d
if __name__ == "__main__":
for name, Dataset in datasets.DATASETS.items():
benchmark_construction(name, Dataset)
|
