1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
#include <gtest/gtest.h>
#include <c10/util/Flags.h>
#include <c10/util/string_view.h>
#include "caffe2/core/blob.h"
#include "caffe2/core/blob_serialization.h"
// NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays
C10_DEFINE_bool(
caffe2_test_generate_unknown_dtype_blob,
false,
"Recompute and log the serialized blob data for the "
"TensorSerialization.TestUnknownDType test");
using namespace caffe2;
namespace {
// This data was computed by serializing a 10-element int32_t tensor,
// but with the data_type field set to 4567. This allows us to test the
// behavior of the code when deserializing data from a future version of the
// code that has new data types that our code does not understand.
constexpr c10::string_view kFutureDtypeBlob(
"\x0a\x09\x74\x65\x73\x74\x5f\x62\x6c\x6f\x62\x12\x06\x54\x65\x6e"
"\x73\x6f\x72\x1a\x28\x08\x0a\x08\x01\x10\xd7\x23\x22\x0a\x00\x01"
"\x02\x03\x04\x05\x06\x07\x08\x09\x3a\x09\x74\x65\x73\x74\x5f\x62"
"\x6c\x6f\x62\x42\x02\x08\x00\x5a\x04\x08\x00\x10\x0a",
61);
// The same tensor with the data_type actually set to TensorProto_DataType_INT32
constexpr c10::string_view kInt32DtypeBlob(
"\x0a\x09\x74\x65\x73\x74\x5f\x62\x6c\x6f\x62\x12\x06\x54\x65\x6e"
"\x73\x6f\x72\x1a\x27\x08\x0a\x08\x01\x10\x02\x22\x0a\x00\x01\x02"
"\x03\x04\x05\x06\x07\x08\x09\x3a\x09\x74\x65\x73\x74\x5f\x62\x6c"
"\x6f\x62\x42\x02\x08\x00\x5a\x04\x08\x00\x10\x0a",
60);
void logBlob(c10::string_view data) {
constexpr size_t kBytesPerLine = 16;
constexpr size_t kCharsPerEncodedByte = 4;
std::vector<char> hexStr;
hexStr.resize((kBytesPerLine * kCharsPerEncodedByte) + 1);
hexStr[kBytesPerLine * kCharsPerEncodedByte] = '\0';
size_t lineIdx = 0;
for (char c : data) {
snprintf(
hexStr.data() + (kCharsPerEncodedByte * lineIdx),
kCharsPerEncodedByte + 1,
"\\x%02x",
static_cast<unsigned int>(c));
++lineIdx;
if (lineIdx >= kBytesPerLine) {
LOG(INFO) << " \"" << hexStr.data() << "\"";
lineIdx = 0;
}
}
if (lineIdx > 0) {
hexStr[lineIdx * kCharsPerEncodedByte] = '\0';
LOG(INFO) << " \"" << hexStr.data() << "\"";
}
}
} // namespace
TEST(TensorSerialization, TestUnknownDType) {
// This code was used to generate the blob data listed above.
constexpr size_t kTestTensorSize = 10;
if (FLAGS_caffe2_test_generate_unknown_dtype_blob) {
Blob blob;
auto* blobTensor = BlobGetMutableTensor(&blob, CPU);
blobTensor->Resize(kTestTensorSize, 1);
auto *tensorData = blobTensor->mutable_data<int32_t>();
for (unsigned n = 0; n < kTestTensorSize; ++n) {
tensorData[n] = n;
}
auto data = SerializeBlob(blob, "test_blob");
LOG(INFO) << "test blob: size=" << data.size();
logBlob(data);
}
// Test deserializing the normal INT32 data,
// just to santity check that deserialization works
Blob i32Blob;
DeserializeBlob(std::string(kInt32DtypeBlob), &i32Blob);
const auto& tensor = BlobGetTensor(i32Blob, c10::DeviceType::CPU);
EXPECT_EQ(kTestTensorSize, tensor.numel());
EXPECT_EQ(TypeMeta::Make<int32_t>(), tensor.dtype());
const auto* tensor_data = tensor.template data<int32_t>();
for (unsigned i = 0; i < kTestTensorSize; ++i) {
EXPECT_EQ(static_cast<float>(i), tensor_data[i]);
}
// Now test deserializing our blob with an unknown data type
Blob futureDtypeBlob;
try {
DeserializeBlob(std::string(kFutureDtypeBlob), &futureDtypeBlob);
FAIL() << "DeserializeBlob() should have failed";
} catch (const std::exception& ex) {
EXPECT_STREQ(
"Cannot deserialize tensor: unrecognized data type", ex.what());
}
}
|
