1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
/*
* SPDX-FileCopyrightText: 2017-2017 CSSlayer <wengxt@gmail.com>
*
* SPDX-License-Identifier: LGPL-2.1-or-later
*/
#include <cstddef>
#include <iostream>
#include <ostream>
#include <string>
#include <string_view>
#include <fcitx-utils/log.h>
#include "libime/core/languagemodel.h"
#include "libime/core/userlanguagemodel.h"
#include "libime/table/tablebaseddictionary.h"
#include "libime/table/tablecontext.h"
#include "libime/table/tabledecoder.h"
#include "libime/table/tableoptions.h"
#include "testdir.h"
#include "testutils.h"
using namespace libime;
class TestLmResolver : public LanguageModelResolver {
public:
TestLmResolver(std::string_view path) : path_(path) {}
protected:
std::string
languageModelFileNameForLanguage(const std::string &language) override {
if (language == "zh_CN") {
return path_;
}
return {};
}
private:
std::string path_;
};
int main() {
fcitx::Log::setLogRule("*=5");
TestLmResolver lmresolver(LIBIME_BINARY_DIR "/data/sc.lm");
auto lm = lmresolver.languageModelFileForLanguage("zh_CN");
TableBasedDictionary dict;
UserLanguageModel model(lm);
dict.load(LIBIME_BINARY_DIR "/data/wbpy.main.dict");
TableOptions options;
options.setLanguageCode("zh_CN");
options.setAutoSelect(true);
options.setAutoSelectLength(-1);
options.setNoMatchAutoSelectLength(-1);
options.setNoSortInputLength(2);
options.setAutoRuleSet({"e2"});
options.setMatchingKey('z');
options.setOrderPolicy(OrderPolicy::Freq);
dict.setTableOptions(options);
TableContext c(dict, model);
auto printTime = [](int t) {
std::cout << "Time: " << t / 1000000.0 << " ms" << std::endl;
};
std::string word;
while (std::cin >> word) {
bool printAll = false;
ScopedNanoTimer t(printTime);
if (word == "back") {
c.backspace();
} else if (word == "reset") {
c.clear();
} else if (word.size() == 1 && ('0' <= word[0] && word[0] <= '9')) {
size_t idx;
if (word[0] == '0') {
idx = 9;
} else {
idx = word[0] - '1';
}
if (c.candidates().size() >= idx) {
c.select(idx);
}
} else if (word.size() == 1 && c.isValidInput(word[0])) {
c.type(word);
} else if (word == "all") {
printAll = true;
} else if (word == "commit") {
c.autoSelect();
c.learn();
c.clear();
}
size_t count = 1;
std::cout << "Preedit: " << c.preedit() << std::endl;
for (const auto &candidate : c.candidates()) {
std::cout << (count % 10) << ": ";
for (const auto *node : candidate.sentence()) {
std::cout
<< node->word() << " "
<< static_cast<const TableLatticeNode *>(node)->code();
}
std::cout << " " << candidate.score() << std::endl;
count++;
if (!printAll && count > 10) {
break;
}
}
}
return 0;
}
|
