1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#include <iostream>
#include <vector>
#include <string>
#include <fstream>
#include <mecab.h>
#include <stdlib.h>
#include "../src/thread.h"
namespace MeCab {
class TaggerThread : public thread {
public:
void run() {
int n = 0;
while (true) {
for (size_t i = 0; i < sentences_->size(); ++i) {
lattice_->set_sentence((*sentences_)[i].c_str());
tagger_->parse(lattice_);
if (n % 100141 == 0) {
std::cout << id_ << " " << n << " parsed" << std::endl;
// std::cout << lattice_->toString();
}
++n;
}
}
}
TaggerThread(std::vector<std::string> *sentences,
MeCab::Model *model,
int id)
: sentences_(sentences), tagger_(0), id_(id) {
tagger_ = model->createTagger();
lattice_ = model->createLattice();
}
~TaggerThread() {
delete tagger_;
delete lattice_;
}
private:
std::vector<std::string> *sentences_;
MeCab::Tagger *tagger_;
MeCab::Lattice *lattice_;
int id_;
};
class ModelUpdater : public thread {
public:
void run() {
const char *kParams[] = {
#ifdef _WIN32
"-d unidic",
"-d ipadic",
"-d jumandic"
#else
"-d /usr/local/lib/mecab/dic/unidic/",
"-d /usr/local/lib/mecab/dic/ipadic/",
"-d /usr/local/lib/mecab/dic/jumandic/"
#endif
};
int i = 0;
while (true) {
#ifdef _WIN32
::Sleep(4000);
#else
sleep(4);
#endif
MeCab::Model *model = MeCab::createModel(kParams[i % 3]);
std::cout << "Updating..." << std::endl;
if (!model_->swap(model)) {
std::cerr << "cannot swap" << std::endl;
exit(-1);
}
std::cout << "Done!" << std::endl;
++i;
}
}
ModelUpdater(MeCab::Model *model) : model_(model) {}
private:
MeCab::Model *model_;
};
}
int main (int argc, char **argv) {
std::ifstream ifs("japanese_sentences.txt");
std::string line;
std::vector<std::string> sentences;
while (std::getline(ifs, line)) {
sentences.push_back(line);
if (sentences.size() == 10000) {
break;
}
}
MeCab::Model *model = MeCab::createModel(argc, argv);
if (!model) {
std::cerr << "model is NULL" << std::endl;
return -1;
}
MeCab::ModelUpdater updater(model);
updater.start();
const int kMaxThreadSize = 8;
std::vector<MeCab::TaggerThread *> threads(kMaxThreadSize);
for (int i = 0; i < kMaxThreadSize; ++i) {
threads[i] = new MeCab::TaggerThread(&sentences, model, i);
}
for (int i = 0; i < kMaxThreadSize; ++i) {
threads[i]->start();
}
for (int i = 0; i < kMaxThreadSize; ++i) {
threads[i]->join();
delete threads[i];
}
updater.join();
delete model;
return 0;
}
|
