File: flexbuffers_test.cpp

package info (click to toggle)
golang-github-google-flatbuffers 24.12.23-1
  • links: PTS, VCS
  • area: main
  • in suites: sid, trixie
  • size: 17,704 kB
  • sloc: cpp: 53,217; python: 6,900; cs: 5,566; java: 4,370; php: 1,460; javascript: 1,061; xml: 1,016; sh: 886; makefile: 13
file content (300 lines) | stat: -rw-r--r-- 12,823 bytes parent folder | download | duplicates (8)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
#include "flexbuffers_test.h"

#include <limits>

#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/idl.h"
#include "is_quiet_nan.h"
#include "test_assert.h"

namespace flatbuffers {
namespace tests {

// Shortcuts for the infinity.
static const auto infinity_d = std::numeric_limits<double>::infinity();

void FlexBuffersTest() {
  flexbuffers::Builder slb(512,
                           flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);

  // Write the equivalent of:
  // { vec: [ -100, "Fred", 4.0, false ], bar: [ 1, 2, 3 ], bar3: [ 1, 2, 3 ],
  // foo: 100, bool: true, mymap: { foo: "Fred" } }

  // It's possible to do this without std::function support as well.
  slb.Map([&]() {
    slb.Vector("vec", [&]() {
      slb += -100;  // Equivalent to slb.Add(-100) or slb.Int(-100);
      slb += "Fred";
      slb.IndirectFloat(4.0f);
      auto i_f = slb.LastValue();
      uint8_t blob[] = { 77 };
      slb.Blob(blob, 1);
      slb += false;
      slb.ReuseValue(i_f);
    });
    int ints[] = { 1, 2, 3 };
    slb.Vector("bar", ints, 3);
    slb.FixedTypedVector("bar3", ints, 3);
    bool bools[] = { true, false, true, false };
    slb.Vector("bools", bools, 4);
    slb.Bool("bool", true);
    slb.Double("foo", 100);
    slb.Map("mymap", [&]() {
      slb.String("foo", "Fred");  // Testing key and string reuse.
    });
  });
  slb.Finish();

  // clang-format off
  #ifdef FLATBUFFERS_TEST_VERBOSE
    for (size_t i = 0; i < slb.GetBuffer().size(); i++)
      printf("%d ", slb.GetBuffer().data()[i]);
    printf("\n");
  #endif
  // clang-format on

  std::vector<uint8_t> reuse_tracker;
  TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
                                    slb.GetBuffer().size(), &reuse_tracker),
          true);

  auto map = flexbuffers::GetRoot(slb.GetBuffer()).AsMap();
  TEST_EQ(map.size(), 7);
  auto vec = map["vec"].AsVector();
  TEST_EQ(vec.size(), 6);
  TEST_EQ(vec[0].AsInt64(), -100);
  TEST_EQ_STR(vec[1].AsString().c_str(), "Fred");
  TEST_EQ(vec[1].AsInt64(), 0);  // Number parsing failed.
  TEST_EQ(vec[2].AsDouble(), 4.0);
  TEST_EQ(vec[2].AsString().IsTheEmptyString(), true);  // Wrong Type.
  TEST_EQ_STR(vec[2].AsString().c_str(), "");     // This still works though.
  TEST_EQ_STR(vec[2].ToString().c_str(), "4.0");  // Or have it converted.
  // Few tests for templated version of As.
  TEST_EQ(vec[0].As<int64_t>(), -100);
  TEST_EQ_STR(vec[1].As<std::string>().c_str(), "Fred");
  TEST_EQ(vec[1].As<int64_t>(), 0);  // Number parsing failed.
  TEST_EQ(vec[2].As<double>(), 4.0);
  // Test that the blob can be accessed.
  TEST_EQ(vec[3].IsBlob(), true);
  auto blob = vec[3].AsBlob();
  TEST_EQ(blob.size(), 1);
  TEST_EQ(blob.data()[0], 77);
  TEST_EQ(vec[4].IsBool(), true);   // Check if type is a bool
  TEST_EQ(vec[4].AsBool(), false);  // Check if value is false
  TEST_EQ(vec[5].AsDouble(), 4.0);  // This is shared with vec[2] !
  auto tvec = map["bar"].AsTypedVector();
  TEST_EQ(tvec.size(), 3);
  TEST_EQ(tvec[2].AsInt8(), 3);
  auto tvec3 = map["bar3"].AsFixedTypedVector();
  TEST_EQ(tvec3.size(), 3);
  TEST_EQ(tvec3[2].AsInt8(), 3);
  TEST_EQ(map["bool"].AsBool(), true);
  auto tvecb = map["bools"].AsTypedVector();
  TEST_EQ(tvecb.ElementType(), flexbuffers::FBT_BOOL);
  TEST_EQ(map["foo"].AsUInt8(), 100);
  TEST_EQ(map["unknown"].IsNull(), true);
  auto mymap = map["mymap"].AsMap();
  // These should be equal by pointer equality, since key and value are shared.
  TEST_EQ(mymap.Keys()[0].AsKey(), map.Keys()[4].AsKey());
  TEST_EQ(mymap.Values()[0].AsString().c_str(), vec[1].AsString().c_str());
  // We can mutate values in the buffer.
  TEST_EQ(vec[0].MutateInt(-99), true);
  TEST_EQ(vec[0].AsInt64(), -99);
  TEST_EQ(vec[1].MutateString("John"), true);  // Size must match.
  TEST_EQ_STR(vec[1].AsString().c_str(), "John");
  TEST_EQ(vec[1].MutateString("Alfred"), false);  // Too long.
  TEST_EQ(vec[2].MutateFloat(2.0f), true);
  TEST_EQ(vec[2].AsFloat(), 2.0f);
  TEST_EQ(vec[2].MutateFloat(3.14159), false);  // Double does not fit in float.
  TEST_EQ(vec[4].AsBool(), false);              // Is false before change
  TEST_EQ(vec[4].MutateBool(true), true);       // Can change a bool
  TEST_EQ(vec[4].AsBool(), true);               // Changed bool is now true

  // Parse from JSON:
  flatbuffers::Parser parser;
  slb.Clear();
  auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\", c: true, d: false }";
  TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
  TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
                                    slb.GetBuffer().size(), &reuse_tracker),
          true);
  auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
  auto jmap = jroot.AsMap();
  auto jvec = jmap["a"].AsVector();
  TEST_EQ(jvec[0].AsInt64(), 123);
  TEST_EQ(jvec[1].AsDouble(), 456.0);
  TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello");
  TEST_EQ(jmap["c"].IsBool(), true);   // Parsed correctly to a bool
  TEST_EQ(jmap["c"].AsBool(), true);   // Parsed correctly to true
  TEST_EQ(jmap["d"].IsBool(), true);   // Parsed correctly to a bool
  TEST_EQ(jmap["d"].AsBool(), false);  // Parsed correctly to false
  // And from FlexBuffer back to JSON:
  auto jsonback = jroot.ToString();
  TEST_EQ_STR(jsontest, jsonback.c_str());
  // With indentation:
  std::string jsonback_indented;
  jroot.ToString(true, false, jsonback_indented, true, 0, "  ");
  auto jsontest_indented =
    "{\n  a: [\n    123,\n    456.0\n  ],\n  b: \"hello\",\n  c: true,\n  d: false\n}";
  TEST_EQ_STR(jsontest_indented, jsonback_indented.c_str());

  slb.Clear();
  slb.Vector([&]() {
    for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
    slb.Vector([&]() {
      for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
      slb.Vector([] {});
    });
  });
  slb.Finish();
  TEST_EQ(slb.GetSize(), 664);
}

void FlexBuffersReuseBugTest() {
  flexbuffers::Builder slb;
  slb.Map([&]() {
    slb.Vector("vec", [&]() {});
    slb.Bool("bool", true);
  });
  slb.Finish();
  std::vector<uint8_t> reuse_tracker;
  // This would fail before, since the reuse_tracker would use the address of
  // the vector reference to check for reuse, but in this case we have an empty
  // vector, and since the size field is before the pointer, its address is the
  // same as thing after it, the key "bool".
  // We fix this by using the address of the size field for tracking reuse.
  TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
                                    slb.GetBuffer().size(), &reuse_tracker),
          true);
}

void FlexBuffersFloatingPointTest() {
#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
  flexbuffers::Builder slb(512,
                           flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
  // Parse floating-point values from JSON:
  flatbuffers::Parser parser;
  slb.Clear();
  auto jsontest =
      "{ a: [1.0, nan, inf, infinity, -inf, +inf, -infinity, 8.0] }";
  TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
  auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
  TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
                                    slb.GetBuffer().size(), nullptr),
          true);
  auto jmap = jroot.AsMap();
  auto jvec = jmap["a"].AsVector();
  TEST_EQ(8, jvec.size());
  TEST_EQ(1.0, jvec[0].AsDouble());
  TEST_ASSERT(is_quiet_nan(jvec[1].AsDouble()));
  TEST_EQ(infinity_d, jvec[2].AsDouble());
  TEST_EQ(infinity_d, jvec[3].AsDouble());
  TEST_EQ(-infinity_d, jvec[4].AsDouble());
  TEST_EQ(+infinity_d, jvec[5].AsDouble());
  TEST_EQ(-infinity_d, jvec[6].AsDouble());
  TEST_EQ(8.0, jvec[7].AsDouble());
#endif
}

void FlexBuffersDeprecatedTest() {
  // FlexBuffers as originally designed had a flaw involving the
  // FBT_VECTOR_STRING datatype, and this test documents/tests the fix for it.
  // Discussion: https://github.com/google/flatbuffers/issues/5627
  flexbuffers::Builder slb;
  // FBT_VECTOR_* are "typed vectors" where all elements are of the same type.
  // Problem is, when storing FBT_STRING elements, it relies on that type to
  // get the bit-width for the size field of the string, which in this case
  // isn't present, and instead defaults to 8-bit. This means that any strings
  // stored inside such a vector, when accessed thru the old API that returns
  // a String reference, will appear to be truncated if the string stored is
  // actually >=256 bytes.
  std::string test_data(300, 'A');
  auto start = slb.StartVector();
  // This one will have a 16-bit size field.
  slb.String(test_data);
  // This one will have an 8-bit size field.
  slb.String("hello");
  // We're asking this to be serialized as a typed vector (true), but not
  // fixed size (false). The type will be FBT_VECTOR_STRING with a bit-width
  // of whatever the offsets in the vector need, the bit-widths of the strings
  // are not stored(!) <- the actual design flaw.
  // Note that even in the fixed code, we continue to serialize the elements of
  // FBT_VECTOR_STRING as FBT_STRING, since there may be old code out there
  // reading new data that we want to continue to function.
  // Thus, FBT_VECTOR_STRING, while deprecated, will always be represented the
  // same way, the fix lies on the reading side.
  slb.EndVector(start, true, false);
  slb.Finish();
  // Verify because why not.
  TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
                                    slb.GetBuffer().size(), nullptr),
          true);
  // So now lets read this data back.
  // For existing data, since we have no way of knowing what the actual
  // bit-width of the size field of the string is, we are going to ignore this
  // field, and instead treat these strings as FBT_KEY (null-terminated), so we
  // can deal with strings of arbitrary length. This of course truncates strings
  // with embedded nulls, but we think that that is preferrable over truncating
  // strings >= 256 bytes.
  auto vec = flexbuffers::GetRoot(slb.GetBuffer()).AsTypedVector();
  // Even though this was serialized as FBT_VECTOR_STRING, it is read as
  // FBT_VECTOR_KEY:
  TEST_EQ(vec.ElementType(), flexbuffers::FBT_KEY);
  // Access the long string. Previously, this would return a string of size 1,
  // since it would read the high-byte of the 16-bit length.
  // This should now correctly test the full 300 bytes, using AsKey():
  TEST_EQ_STR(vec[0].AsKey(), test_data.c_str());
  // Old code that called AsString will continue to work, as the String
  // accessor objects now use a cached size that can come from a key as well.
  TEST_EQ_STR(vec[0].AsString().c_str(), test_data.c_str());
  // Short strings work as before:
  TEST_EQ_STR(vec[1].AsKey(), "hello");
  TEST_EQ_STR(vec[1].AsString().c_str(), "hello");
  // So, while existing code and data mostly "just work" with the fixes applied
  // to AsTypedVector and AsString, what do you do going forward?
  // Code accessing existing data doesn't necessarily need to change, though
  // you could consider using AsKey instead of AsString for a) documenting
  // that you are accessing keys, or b) a speedup if you don't actually use
  // the string size.
  // For new data, or data that doesn't need to be backwards compatible,
  // instead serialize as FBT_VECTOR (call EndVector with typed = false, then
  // read elements with AsString), or, for maximum compactness, use
  // FBT_VECTOR_KEY (call slb.Key above instead, read with AsKey or AsString).
}

void ParseFlexbuffersFromJsonWithNullTest() {
  // Test nulls are handled appropriately through flexbuffers to exercise other
  // code paths of ParseSingleValue in the optional scalars change.
  // TODO(cneo): Json -> Flatbuffers test once some language can generate code
  // with optional scalars.
  {
    char json[] = "{\"opt_field\": 123 }";
    flatbuffers::Parser parser;
    flexbuffers::Builder flexbuild;
    parser.ParseFlexBuffer(json, nullptr, &flexbuild);
    auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
    TEST_EQ(root.AsMap()["opt_field"].AsInt64(), 123);
  }
  {
    char json[] = "{\"opt_field\": 123.4 }";
    flatbuffers::Parser parser;
    flexbuffers::Builder flexbuild;
    parser.ParseFlexBuffer(json, nullptr, &flexbuild);
    auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
    TEST_EQ(root.AsMap()["opt_field"].AsDouble(), 123.4);
  }
  {
    char json[] = "{\"opt_field\": null }";
    flatbuffers::Parser parser;
    flexbuffers::Builder flexbuild;
    parser.ParseFlexBuffer(json, nullptr, &flexbuild);
    auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
    TEST_ASSERT(!root.AsMap().IsTheEmptyMap());
    TEST_ASSERT(root.AsMap()["opt_field"].IsNull());
    TEST_EQ(root.ToString(), std::string("{ opt_field: null }"));
  }
}

}  // namespace tests
}  // namespace flatbuffers