1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
#include "test.h"
#include "tokenizer.h"
TEST_CASE("trim") {
CHECK(Tokenizer::trim(" abc ") == "abc");
CHECK(Tokenizer::trim("abcd ") == "abcd");
CHECK(Tokenizer::trim("ab cd") == "ab cd");
CHECK(Tokenizer::trim("abc") == "abc");
CHECK(Tokenizer::trim(" bcd") == "bcd");
CHECK(Tokenizer::trim(" \tbcd\t") == "bcd");
CHECK(Tokenizer::trim("a,b,c", "ac") == ",b,");
CHECK(Tokenizer::trim("a b c", "ac") == " b ");
CHECK(Tokenizer::trim("a,b,c", ",ac") == "b");
}
TEST_CASE("extract") {
int i, j;
float f;
string s, t;
bool b;
int ret;
ret = Tokenizer::extract("%=%", "val=true", &s, &b);
CHECK(ret == 2);
CHECK(s == "val");
CHECK(b);
ret = Tokenizer::extract("%=%", "0=false", &s, &b);
CHECK(ret == 2);
CHECK(s == "0");
CHECK(!b);
ret = Tokenizer::extract("%+2=%", "1+2=3 ", &i, &j);
CHECK(ret == 2);
CHECK(i == 1);
CHECK(j == 3);
j = -1;
ret = Tokenizer::extract("%-2=%", "1+2=3 ", &i, &j);
CHECK(ret == 1);
CHECK(i == 1);
CHECK(j == -1);
ret = Tokenizer::extract("%&%=%&%", "query?a=b&c=0.34&d=e",
nullptr, &s, &f, nullptr);
CHECK(ret == 4);
CHECK(s == "c");
CHECK(f == 0.34f);
ret = Tokenizer::extract("function(%, %)", "function(parm1, parm2)",
&s, &t);
CHECK(ret == 2);
CHECK(s == "parm1");
CHECK(t == "parm2");
}
TEST_CASE("annotate quote") {
string rt = "__________\"'''''''''''\"_________________";
string in = "this is a \"string with\" some quoted part";
string out;
Tokenizer::annotate_quote(in, &out);
CHECK(out == rt);
}
TEST_CASE("annotate multi quote") {
string rt = "________\"'\"________\"''''\"_\"'''''''''''\"";
string in = "this is \"a\" string \"with\" \"some quotes\"";
string out;
Tokenizer::annotate_quote(in, &out);
CHECK(out == rt);
}
TEST_CASE("split mind quote") {
string in = "this,can,\"be,split\",fine";
vector<string> out = {"this", "can", "\"be,split\"", "fine"};
vector<string_view> ret;
Tokenizer::split_mind_quote(in, ",", &ret);
CHECK(ret.size() == out.size());
for (size_t i = 0; i < ret.size(); ++i) {
CHECK(ret[i] == out[i]);
}
}
TEST_CASE("split mind quote no quotes") {
string in = "this,can,be,split,fine";
vector<string> out = {"this", "can", "be", "split", "fine"};
vector<string_view> ret;
Tokenizer::split_mind_quote(in, ",", &ret);
CHECK(ret.size() == out.size());
for (size_t i = 0; i < ret.size(); ++i) {
CHECK(ret[i] == out[i]);
}
}
TEST_CASE("split mind quote multi") {
string in = "this-\"can-be-split\"-\"just-fine\"";
vector<string> out = {"this", "\"can-be-split\"", "\"just-fine\""};
vector<string_view> ret;
Tokenizer::split_mind_quote(in, "-", &ret);
CHECK(ret.size() == out.size());
for (size_t i = 0; i < ret.size(); ++i) {
CHECK(ret[i] == out[i]);
}
}
/* TODO: tests for the rest of tokenizer, however this covers the functions
* logserver uses */
|
