1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
|
#include "flexbuffers_test.h"
#include <limits>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/idl.h"
#include "is_quiet_nan.h"
#include "test_assert.h"
namespace flatbuffers {
namespace tests {
// Shortcuts for the infinity.
static const auto infinity_d = std::numeric_limits<double>::infinity();
void FlexBuffersTest() {
flexbuffers::Builder slb(512,
flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
// Write the equivalent of:
// { vec: [ -100, "Fred", 4.0, false ], bar: [ 1, 2, 3 ], bar3: [ 1, 2, 3 ],
// foo: 100, bool: true, mymap: { foo: "Fred" } }
// It's possible to do this without std::function support as well.
slb.Map([&]() {
slb.Vector("vec", [&]() {
slb += -100; // Equivalent to slb.Add(-100) or slb.Int(-100);
slb += "Fred";
slb.IndirectFloat(4.0f);
auto i_f = slb.LastValue();
uint8_t blob[] = { 77 };
slb.Blob(blob, 1);
slb += false;
slb.ReuseValue(i_f);
});
int ints[] = { 1, 2, 3 };
slb.Vector("bar", ints, 3);
slb.FixedTypedVector("bar3", ints, 3);
bool bools[] = { true, false, true, false };
slb.Vector("bools", bools, 4);
slb.Bool("bool", true);
slb.Double("foo", 100);
slb.Map("mymap", [&]() {
slb.String("foo", "Fred"); // Testing key and string reuse.
});
});
slb.Finish();
// clang-format off
#ifdef FLATBUFFERS_TEST_VERBOSE
for (size_t i = 0; i < slb.GetBuffer().size(); i++)
printf("%d ", slb.GetBuffer().data()[i]);
printf("\n");
#endif
// clang-format on
std::vector<uint8_t> reuse_tracker;
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
slb.GetBuffer().size(), &reuse_tracker),
true);
auto map = flexbuffers::GetRoot(slb.GetBuffer()).AsMap();
TEST_EQ(map.size(), 7);
auto vec = map["vec"].AsVector();
TEST_EQ(vec.size(), 6);
TEST_EQ(vec[0].AsInt64(), -100);
TEST_EQ_STR(vec[1].AsString().c_str(), "Fred");
TEST_EQ(vec[1].AsInt64(), 0); // Number parsing failed.
TEST_EQ(vec[2].AsDouble(), 4.0);
TEST_EQ(vec[2].AsString().IsTheEmptyString(), true); // Wrong Type.
TEST_EQ_STR(vec[2].AsString().c_str(), ""); // This still works though.
TEST_EQ_STR(vec[2].ToString().c_str(), "4.0"); // Or have it converted.
// Few tests for templated version of As.
TEST_EQ(vec[0].As<int64_t>(), -100);
TEST_EQ_STR(vec[1].As<std::string>().c_str(), "Fred");
TEST_EQ(vec[1].As<int64_t>(), 0); // Number parsing failed.
TEST_EQ(vec[2].As<double>(), 4.0);
// Test that the blob can be accessed.
TEST_EQ(vec[3].IsBlob(), true);
auto blob = vec[3].AsBlob();
TEST_EQ(blob.size(), 1);
TEST_EQ(blob.data()[0], 77);
TEST_EQ(vec[4].IsBool(), true); // Check if type is a bool
TEST_EQ(vec[4].AsBool(), false); // Check if value is false
TEST_EQ(vec[5].AsDouble(), 4.0); // This is shared with vec[2] !
auto tvec = map["bar"].AsTypedVector();
TEST_EQ(tvec.size(), 3);
TEST_EQ(tvec[2].AsInt8(), 3);
auto tvec3 = map["bar3"].AsFixedTypedVector();
TEST_EQ(tvec3.size(), 3);
TEST_EQ(tvec3[2].AsInt8(), 3);
TEST_EQ(map["bool"].AsBool(), true);
auto tvecb = map["bools"].AsTypedVector();
TEST_EQ(tvecb.ElementType(), flexbuffers::FBT_BOOL);
TEST_EQ(map["foo"].AsUInt8(), 100);
TEST_EQ(map["unknown"].IsNull(), true);
auto mymap = map["mymap"].AsMap();
// These should be equal by pointer equality, since key and value are shared.
TEST_EQ(mymap.Keys()[0].AsKey(), map.Keys()[4].AsKey());
TEST_EQ(mymap.Values()[0].AsString().c_str(), vec[1].AsString().c_str());
// We can mutate values in the buffer.
TEST_EQ(vec[0].MutateInt(-99), true);
TEST_EQ(vec[0].AsInt64(), -99);
TEST_EQ(vec[1].MutateString("John"), true); // Size must match.
TEST_EQ_STR(vec[1].AsString().c_str(), "John");
TEST_EQ(vec[1].MutateString("Alfred"), false); // Too long.
TEST_EQ(vec[2].MutateFloat(2.0f), true);
TEST_EQ(vec[2].AsFloat(), 2.0f);
TEST_EQ(vec[2].MutateFloat(3.14159), false); // Double does not fit in float.
TEST_EQ(vec[4].AsBool(), false); // Is false before change
TEST_EQ(vec[4].MutateBool(true), true); // Can change a bool
TEST_EQ(vec[4].AsBool(), true); // Changed bool is now true
// Parse from JSON:
flatbuffers::Parser parser;
slb.Clear();
auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\", c: true, d: false }";
TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
slb.GetBuffer().size(), &reuse_tracker),
true);
auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
auto jmap = jroot.AsMap();
auto jvec = jmap["a"].AsVector();
TEST_EQ(jvec[0].AsInt64(), 123);
TEST_EQ(jvec[1].AsDouble(), 456.0);
TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello");
TEST_EQ(jmap["c"].IsBool(), true); // Parsed correctly to a bool
TEST_EQ(jmap["c"].AsBool(), true); // Parsed correctly to true
TEST_EQ(jmap["d"].IsBool(), true); // Parsed correctly to a bool
TEST_EQ(jmap["d"].AsBool(), false); // Parsed correctly to false
// And from FlexBuffer back to JSON:
auto jsonback = jroot.ToString();
TEST_EQ_STR(jsontest, jsonback.c_str());
// With indentation:
std::string jsonback_indented;
jroot.ToString(true, false, jsonback_indented, true, 0, " ");
auto jsontest_indented =
"{\n a: [\n 123,\n 456.0\n ],\n b: \"hello\",\n c: true,\n d: false\n}";
TEST_EQ_STR(jsontest_indented, jsonback_indented.c_str());
slb.Clear();
slb.Vector([&]() {
for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
slb.Vector([&]() {
for (int i = 0; i < 130; ++i) slb.Add(static_cast<uint8_t>(255));
slb.Vector([] {});
});
});
slb.Finish();
TEST_EQ(slb.GetSize(), 664);
}
void FlexBuffersReuseBugTest() {
flexbuffers::Builder slb;
slb.Map([&]() {
slb.Vector("vec", [&]() {});
slb.Bool("bool", true);
});
slb.Finish();
std::vector<uint8_t> reuse_tracker;
// This would fail before, since the reuse_tracker would use the address of
// the vector reference to check for reuse, but in this case we have an empty
// vector, and since the size field is before the pointer, its address is the
// same as thing after it, the key "bool".
// We fix this by using the address of the size field for tracking reuse.
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
slb.GetBuffer().size(), &reuse_tracker),
true);
}
void FlexBuffersFloatingPointTest() {
#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
flexbuffers::Builder slb(512,
flexbuffers::BUILDER_FLAG_SHARE_KEYS_AND_STRINGS);
// Parse floating-point values from JSON:
flatbuffers::Parser parser;
slb.Clear();
auto jsontest =
"{ a: [1.0, nan, inf, infinity, -inf, +inf, -infinity, 8.0] }";
TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb), true);
auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
slb.GetBuffer().size(), nullptr),
true);
auto jmap = jroot.AsMap();
auto jvec = jmap["a"].AsVector();
TEST_EQ(8, jvec.size());
TEST_EQ(1.0, jvec[0].AsDouble());
TEST_ASSERT(is_quiet_nan(jvec[1].AsDouble()));
TEST_EQ(infinity_d, jvec[2].AsDouble());
TEST_EQ(infinity_d, jvec[3].AsDouble());
TEST_EQ(-infinity_d, jvec[4].AsDouble());
TEST_EQ(+infinity_d, jvec[5].AsDouble());
TEST_EQ(-infinity_d, jvec[6].AsDouble());
TEST_EQ(8.0, jvec[7].AsDouble());
#endif
}
void FlexBuffersDeprecatedTest() {
// FlexBuffers as originally designed had a flaw involving the
// FBT_VECTOR_STRING datatype, and this test documents/tests the fix for it.
// Discussion: https://github.com/google/flatbuffers/issues/5627
flexbuffers::Builder slb;
// FBT_VECTOR_* are "typed vectors" where all elements are of the same type.
// Problem is, when storing FBT_STRING elements, it relies on that type to
// get the bit-width for the size field of the string, which in this case
// isn't present, and instead defaults to 8-bit. This means that any strings
// stored inside such a vector, when accessed thru the old API that returns
// a String reference, will appear to be truncated if the string stored is
// actually >=256 bytes.
std::string test_data(300, 'A');
auto start = slb.StartVector();
// This one will have a 16-bit size field.
slb.String(test_data);
// This one will have an 8-bit size field.
slb.String("hello");
// We're asking this to be serialized as a typed vector (true), but not
// fixed size (false). The type will be FBT_VECTOR_STRING with a bit-width
// of whatever the offsets in the vector need, the bit-widths of the strings
// are not stored(!) <- the actual design flaw.
// Note that even in the fixed code, we continue to serialize the elements of
// FBT_VECTOR_STRING as FBT_STRING, since there may be old code out there
// reading new data that we want to continue to function.
// Thus, FBT_VECTOR_STRING, while deprecated, will always be represented the
// same way, the fix lies on the reading side.
slb.EndVector(start, true, false);
slb.Finish();
// Verify because why not.
TEST_EQ(flexbuffers::VerifyBuffer(slb.GetBuffer().data(),
slb.GetBuffer().size(), nullptr),
true);
// So now lets read this data back.
// For existing data, since we have no way of knowing what the actual
// bit-width of the size field of the string is, we are going to ignore this
// field, and instead treat these strings as FBT_KEY (null-terminated), so we
// can deal with strings of arbitrary length. This of course truncates strings
// with embedded nulls, but we think that that is preferrable over truncating
// strings >= 256 bytes.
auto vec = flexbuffers::GetRoot(slb.GetBuffer()).AsTypedVector();
// Even though this was serialized as FBT_VECTOR_STRING, it is read as
// FBT_VECTOR_KEY:
TEST_EQ(vec.ElementType(), flexbuffers::FBT_KEY);
// Access the long string. Previously, this would return a string of size 1,
// since it would read the high-byte of the 16-bit length.
// This should now correctly test the full 300 bytes, using AsKey():
TEST_EQ_STR(vec[0].AsKey(), test_data.c_str());
// Old code that called AsString will continue to work, as the String
// accessor objects now use a cached size that can come from a key as well.
TEST_EQ_STR(vec[0].AsString().c_str(), test_data.c_str());
// Short strings work as before:
TEST_EQ_STR(vec[1].AsKey(), "hello");
TEST_EQ_STR(vec[1].AsString().c_str(), "hello");
// So, while existing code and data mostly "just work" with the fixes applied
// to AsTypedVector and AsString, what do you do going forward?
// Code accessing existing data doesn't necessarily need to change, though
// you could consider using AsKey instead of AsString for a) documenting
// that you are accessing keys, or b) a speedup if you don't actually use
// the string size.
// For new data, or data that doesn't need to be backwards compatible,
// instead serialize as FBT_VECTOR (call EndVector with typed = false, then
// read elements with AsString), or, for maximum compactness, use
// FBT_VECTOR_KEY (call slb.Key above instead, read with AsKey or AsString).
}
void ParseFlexbuffersFromJsonWithNullTest() {
// Test nulls are handled appropriately through flexbuffers to exercise other
// code paths of ParseSingleValue in the optional scalars change.
// TODO(cneo): Json -> Flatbuffers test once some language can generate code
// with optional scalars.
{
char json[] = "{\"opt_field\": 123 }";
flatbuffers::Parser parser;
flexbuffers::Builder flexbuild;
parser.ParseFlexBuffer(json, nullptr, &flexbuild);
auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
TEST_EQ(root.AsMap()["opt_field"].AsInt64(), 123);
}
{
char json[] = "{\"opt_field\": 123.4 }";
flatbuffers::Parser parser;
flexbuffers::Builder flexbuild;
parser.ParseFlexBuffer(json, nullptr, &flexbuild);
auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
TEST_EQ(root.AsMap()["opt_field"].AsDouble(), 123.4);
}
{
char json[] = "{\"opt_field\": null }";
flatbuffers::Parser parser;
flexbuffers::Builder flexbuild;
parser.ParseFlexBuffer(json, nullptr, &flexbuild);
auto root = flexbuffers::GetRoot(flexbuild.GetBuffer());
TEST_ASSERT(!root.AsMap().IsTheEmptyMap());
TEST_ASSERT(root.AsMap()["opt_field"].IsNull());
TEST_EQ(root.ToString(), std::string("{ opt_field: null }"));
}
}
} // namespace tests
} // namespace flatbuffers
|