1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
#include <iostream>
#include <unistd.h>
#include "simdjson.h"
#ifdef __linux__
#include "linux-perf-events.h"
#endif
size_t count_nonasciibytes(const uint8_t *input, size_t length) {
size_t count = 0;
for (size_t i = 0; i < length; i++) {
count += input[i] >> 7;
}
return count;
}
size_t count_backslash(const uint8_t *input, size_t length) {
size_t count = 0;
for (size_t i = 0; i < length; i++) {
count += (input[i] == '\\') ? 1 : 0;
}
return count;
}
struct stat_s {
size_t integer_count;
size_t float_count;
size_t string_count;
size_t backslash_count;
size_t non_ascii_byte_count;
size_t object_count;
size_t array_count;
size_t null_count;
size_t true_count;
size_t false_count;
size_t byte_count;
size_t structural_indexes_count;
bool valid;
};
using stat_t = struct stat_s;
simdjson_inline void simdjson_process_atom(stat_t &s,
simdjson::dom::element element) {
if (element.is<int64_t>()) {
s.integer_count++;
} else if(element.is<std::string_view>()) {
s.string_count++;
} else if(element.is<double>()) {
s.float_count++;
} else if (element.is<bool>()) {
bool v;
simdjson::error_code error;
if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); }
if (v) {
s.true_count++;
} else {
s.false_count++;
}
} else if (element.is_null()) {
s.null_count++;
}
}
void simdjson_recurse(stat_t &s, simdjson::dom::element element) {
simdjson::error_code error;
if (element.is<simdjson::dom::array>()) {
s.array_count++;
simdjson::dom::array array;
if ((error = element.get(array))) { std::cerr << error << std::endl; abort(); }
for (auto child : array) {
if (child.is<simdjson::dom::array>() || child.is<simdjson::dom::object>()) {
simdjson_recurse(s, child);
} else {
simdjson_process_atom(s, child);
}
}
} else if (element.is<simdjson::dom::object>()) {
s.object_count++;
simdjson::dom::object object;
if ((error = element.get(object))) { std::cerr << error << std::endl; abort(); }
for (auto field : object) {
s.string_count++; // for key
if (field.value.is<simdjson::dom::array>() || field.value.is<simdjson::dom::object>()) {
simdjson_recurse(s, field.value);
} else {
simdjson_process_atom(s, field.value);
}
}
} else {
simdjson_process_atom(s, element);
}
}
stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
stat_t answer{};
simdjson::dom::parser parser;
simdjson::dom::element doc;
auto error = parser.parse(p).get(doc);
if (error) {
answer.valid = false;
return answer;
}
answer.valid = true;
answer.backslash_count =
count_backslash(reinterpret_cast<const uint8_t *>(p.data()), p.size());
answer.non_ascii_byte_count = count_nonasciibytes(
reinterpret_cast<const uint8_t *>(p.data()), p.size());
answer.byte_count = p.size();
answer.structural_indexes_count = parser.implementation->n_structural_indexes;
simdjson_recurse(answer, doc);
return answer;
}
int main(int argc, char *argv[]) {
#ifndef _MSC_VER
int c;
while ((c = getopt(argc, argv, "")) != -1) {
switch (c) {
default:
abort();
}
}
#else
int optind = 1;
#endif
if (optind >= argc) {
std::cerr << "Reads json, prints stats. " << std::endl;
std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
exit(1);
}
const char *filename = argv[optind];
if (optind + 1 < argc) {
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
<< std::endl;
}
simdjson::padded_string p;
auto error = simdjson::padded_string::load(filename).get(p);
if (error) {
std::cerr << "Could not load the file " << filename << std::endl;
return EXIT_FAILURE;
}
stat_t s = simdjson_compute_stats(p);
if (!s.valid) {
std::cerr << "not a valid JSON" << std::endl;
return EXIT_FAILURE;
}
printf("# integer_count float_count string_count backslash_count "
"non_ascii_byte_count object_count array_count null_count true_count "
"false_count byte_count structural_indexes_count ");
#ifdef __linux__
printf(" stage1_cycle_count stage1_instruction_count stage2_cycle_count "
" stage2_instruction_count stage3_cycle_count "
"stage3_instruction_count ");
#else
printf("(you are not under linux, so perf counters are disaabled)");
#endif
printf("\n");
printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu ", s.integer_count,
s.float_count, s.string_count, s.backslash_count,
s.non_ascii_byte_count, s.object_count, s.array_count, s.null_count,
s.true_count, s.false_count, s.byte_count, s.structural_indexes_count);
#ifdef __linux__
simdjson::dom::parser parser;
simdjson::error_code alloc_error = parser.allocate(p.size());
if (alloc_error) {
std::cerr << alloc_error << std::endl;
return EXIT_FAILURE;
}
const uint32_t iterations = p.size() < 1 * 1000 * 1000 ? 1000 : 50;
std::vector<int> evts;
evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
unsigned long cy1 = 0, cy2 = 0;
unsigned long cl1 = 0, cl2 = 0;
std::vector<unsigned long long> results;
results.resize(evts.size());
for (uint32_t i = 0; i < iterations; i++) {
unified.start();
// The default template is simdjson::architecture::NATIVE.
bool isok = (parser.implementation->stage1((const uint8_t *)p.data(), p.size(), simdjson::stage1_mode::regular) == simdjson::SUCCESS);
unified.end(results);
cy1 += results[0];
cl1 += results[1];
unified.start();
isok = isok && (parser.implementation->stage2(parser.doc) == simdjson::SUCCESS);
unified.end(results);
cy2 += results[0];
cl2 += results[1];
if (!isok) {
std::cerr << "failure?" << std::endl;
}
}
printf("%f %f %f %f ", static_cast<double>(cy1) / static_cast<double>(iterations), static_cast<double>(cl1) / static_cast<double>(iterations),
static_cast<double>(cy2) / static_cast<double>(iterations), static_cast<double>(cl2) / static_cast<double>(iterations));
#endif // __linux__
printf("\n");
return EXIT_SUCCESS;
}
|
