1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
|
#include "llvm/ProfileData/MemProf.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/Function.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/EndianStream.h"
namespace llvm {
namespace memprof {
MemProfSchema getFullSchema() {
MemProfSchema List;
#define MIBEntryDef(NameTag, Name, Type) List.push_back(Meta::Name);
#include "llvm/ProfileData/MIBEntryDef.inc"
#undef MIBEntryDef
return List;
}
MemProfSchema getHotColdSchema() {
return {Meta::AllocCount, Meta::TotalSize, Meta::TotalLifetime,
Meta::TotalLifetimeAccessDensity};
}
static size_t serializedSizeV2(const IndexedAllocationInfo &IAI,
const MemProfSchema &Schema) {
size_t Size = 0;
// The CallStackId
Size += sizeof(CallStackId);
// The size of the payload.
Size += PortableMemInfoBlock::serializedSize(Schema);
return Size;
}
static size_t serializedSizeV3(const IndexedAllocationInfo &IAI,
const MemProfSchema &Schema) {
size_t Size = 0;
// The linear call stack ID.
Size += sizeof(LinearCallStackId);
// The size of the payload.
Size += PortableMemInfoBlock::serializedSize(Schema);
return Size;
}
size_t IndexedAllocationInfo::serializedSize(const MemProfSchema &Schema,
IndexedVersion Version) const {
switch (Version) {
case Version2:
return serializedSizeV2(*this, Schema);
// Combine V3 and V4 as the size calculation is the same
case Version3:
case Version4:
return serializedSizeV3(*this, Schema);
}
llvm_unreachable("unsupported MemProf version");
}
static size_t serializedSizeV2(const IndexedMemProfRecord &Record,
const MemProfSchema &Schema) {
// The number of alloc sites to serialize.
size_t Result = sizeof(uint64_t);
for (const IndexedAllocationInfo &N : Record.AllocSites)
Result += N.serializedSize(Schema, Version2);
// The number of callsites we have information for.
Result += sizeof(uint64_t);
// The CallStackId
Result += Record.CallSites.size() * sizeof(CallStackId);
return Result;
}
static size_t serializedSizeV3(const IndexedMemProfRecord &Record,
const MemProfSchema &Schema) {
// The number of alloc sites to serialize.
size_t Result = sizeof(uint64_t);
for (const IndexedAllocationInfo &N : Record.AllocSites)
Result += N.serializedSize(Schema, Version3);
// The number of callsites we have information for.
Result += sizeof(uint64_t);
// The linear call stack ID.
// Note: V3 only stored the LinearCallStackId per call site.
Result += Record.CallSites.size() * sizeof(LinearCallStackId);
return Result;
}
static size_t serializedSizeV4(const IndexedMemProfRecord &Record,
const MemProfSchema &Schema) {
// The number of alloc sites to serialize.
size_t Result = sizeof(uint64_t);
for (const IndexedAllocationInfo &N : Record.AllocSites)
Result += N.serializedSize(Schema, Version4);
// The number of callsites we have information for.
Result += sizeof(uint64_t);
for (const auto &CS : Record.CallSites)
Result += sizeof(LinearCallStackId) + sizeof(uint64_t) +
CS.CalleeGuids.size() * sizeof(GlobalValue::GUID);
return Result;
}
size_t IndexedMemProfRecord::serializedSize(const MemProfSchema &Schema,
IndexedVersion Version) const {
switch (Version) {
case Version2:
return serializedSizeV2(*this, Schema);
case Version3:
return serializedSizeV3(*this, Schema);
case Version4:
return serializedSizeV4(*this, Schema);
}
llvm_unreachable("unsupported MemProf version");
}
static void serializeV2(const IndexedMemProfRecord &Record,
const MemProfSchema &Schema, raw_ostream &OS) {
using namespace support;
endian::Writer LE(OS, llvm::endianness::little);
LE.write<uint64_t>(Record.AllocSites.size());
for (const IndexedAllocationInfo &N : Record.AllocSites) {
LE.write<CallStackId>(N.CSId);
N.Info.serialize(Schema, OS);
}
// Related contexts.
LE.write<uint64_t>(Record.CallSites.size());
for (const auto &CS : Record.CallSites)
LE.write<CallStackId>(CS.CSId);
}
static void serializeV3(
const IndexedMemProfRecord &Record, const MemProfSchema &Schema,
raw_ostream &OS,
llvm::DenseMap<CallStackId, LinearCallStackId> &MemProfCallStackIndexes) {
using namespace support;
endian::Writer LE(OS, llvm::endianness::little);
LE.write<uint64_t>(Record.AllocSites.size());
for (const IndexedAllocationInfo &N : Record.AllocSites) {
assert(MemProfCallStackIndexes.contains(N.CSId));
LE.write<LinearCallStackId>(MemProfCallStackIndexes[N.CSId]);
N.Info.serialize(Schema, OS);
}
// Related contexts.
LE.write<uint64_t>(Record.CallSites.size());
for (const auto &CS : Record.CallSites) {
assert(MemProfCallStackIndexes.contains(CS.CSId));
LE.write<LinearCallStackId>(MemProfCallStackIndexes[CS.CSId]);
}
}
static void serializeV4(
const IndexedMemProfRecord &Record, const MemProfSchema &Schema,
raw_ostream &OS,
llvm::DenseMap<CallStackId, LinearCallStackId> &MemProfCallStackIndexes) {
using namespace support;
endian::Writer LE(OS, llvm::endianness::little);
LE.write<uint64_t>(Record.AllocSites.size());
for (const IndexedAllocationInfo &N : Record.AllocSites) {
assert(MemProfCallStackIndexes.contains(N.CSId));
LE.write<LinearCallStackId>(MemProfCallStackIndexes[N.CSId]);
N.Info.serialize(Schema, OS);
}
// Related contexts.
LE.write<uint64_t>(Record.CallSites.size());
for (const auto &CS : Record.CallSites) {
assert(MemProfCallStackIndexes.contains(CS.CSId));
LE.write<LinearCallStackId>(MemProfCallStackIndexes[CS.CSId]);
LE.write<uint64_t>(CS.CalleeGuids.size());
for (const auto &Guid : CS.CalleeGuids)
LE.write<GlobalValue::GUID>(Guid);
}
}
void IndexedMemProfRecord::serialize(
const MemProfSchema &Schema, raw_ostream &OS, IndexedVersion Version,
llvm::DenseMap<CallStackId, LinearCallStackId> *MemProfCallStackIndexes)
const {
switch (Version) {
case Version2:
serializeV2(*this, Schema, OS);
return;
case Version3:
serializeV3(*this, Schema, OS, *MemProfCallStackIndexes);
return;
case Version4:
serializeV4(*this, Schema, OS, *MemProfCallStackIndexes);
return;
}
llvm_unreachable("unsupported MemProf version");
}
static IndexedMemProfRecord deserializeV2(const MemProfSchema &Schema,
const unsigned char *Ptr) {
using namespace support;
IndexedMemProfRecord Record;
// Read the meminfo nodes.
const uint64_t NumNodes =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
Record.AllocSites.reserve(NumNodes);
for (uint64_t I = 0; I < NumNodes; I++) {
IndexedAllocationInfo Node;
Node.CSId = endian::readNext<CallStackId, llvm::endianness::little>(Ptr);
Node.Info.deserialize(Schema, Ptr);
Ptr += PortableMemInfoBlock::serializedSize(Schema);
Record.AllocSites.push_back(Node);
}
// Read the callsite information.
const uint64_t NumCtxs =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
Record.CallSites.reserve(NumCtxs);
for (uint64_t J = 0; J < NumCtxs; J++) {
CallStackId CSId =
endian::readNext<CallStackId, llvm::endianness::little>(Ptr);
Record.CallSites.emplace_back(CSId);
}
return Record;
}
static IndexedMemProfRecord deserializeV3(const MemProfSchema &Schema,
const unsigned char *Ptr) {
using namespace support;
IndexedMemProfRecord Record;
// Read the meminfo nodes.
const uint64_t NumNodes =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
Record.AllocSites.reserve(NumNodes);
const size_t SerializedSize = PortableMemInfoBlock::serializedSize(Schema);
for (uint64_t I = 0; I < NumNodes; I++) {
IndexedAllocationInfo Node;
Node.CSId =
endian::readNext<LinearCallStackId, llvm::endianness::little>(Ptr);
Node.Info.deserialize(Schema, Ptr);
Ptr += SerializedSize;
Record.AllocSites.push_back(Node);
}
// Read the callsite information.
const uint64_t NumCtxs =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
Record.CallSites.reserve(NumCtxs);
for (uint64_t J = 0; J < NumCtxs; J++) {
// We are storing LinearCallStackId in CallSiteIds, which is a vector of
// CallStackId. Assert that CallStackId is no smaller than
// LinearCallStackId.
static_assert(sizeof(LinearCallStackId) <= sizeof(CallStackId));
LinearCallStackId CSId =
endian::readNext<LinearCallStackId, llvm::endianness::little>(Ptr);
Record.CallSites.emplace_back(CSId);
}
return Record;
}
static IndexedMemProfRecord deserializeV4(const MemProfSchema &Schema,
const unsigned char *Ptr) {
using namespace support;
IndexedMemProfRecord Record;
// Read the meminfo nodes.
const uint64_t NumNodes =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
Record.AllocSites.reserve(NumNodes);
const size_t SerializedSize = PortableMemInfoBlock::serializedSize(Schema);
for (uint64_t I = 0; I < NumNodes; I++) {
IndexedAllocationInfo Node;
Node.CSId =
endian::readNext<LinearCallStackId, llvm::endianness::little>(Ptr);
Node.Info.deserialize(Schema, Ptr);
Ptr += SerializedSize;
Record.AllocSites.push_back(Node);
}
// Read the callsite information.
const uint64_t NumCtxs =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
Record.CallSites.reserve(NumCtxs);
for (uint64_t J = 0; J < NumCtxs; J++) {
static_assert(sizeof(LinearCallStackId) <= sizeof(CallStackId));
LinearCallStackId CSId =
endian::readNext<LinearCallStackId, llvm::endianness::little>(Ptr);
const uint64_t NumGuids =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
SmallVector<GlobalValue::GUID, 1> Guids;
Guids.reserve(NumGuids);
for (uint64_t K = 0; K < NumGuids; ++K)
Guids.push_back(
endian::readNext<GlobalValue::GUID, llvm::endianness::little>(Ptr));
Record.CallSites.emplace_back(CSId, std::move(Guids));
}
return Record;
}
IndexedMemProfRecord
IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
const unsigned char *Ptr,
IndexedVersion Version) {
switch (Version) {
case Version2:
return deserializeV2(Schema, Ptr);
case Version3:
return deserializeV3(Schema, Ptr);
case Version4:
return deserializeV4(Schema, Ptr);
}
llvm_unreachable("unsupported MemProf version");
}
MemProfRecord IndexedMemProfRecord::toMemProfRecord(
llvm::function_ref<std::vector<Frame>(const CallStackId)> Callback) const {
MemProfRecord Record;
Record.AllocSites.reserve(AllocSites.size());
for (const IndexedAllocationInfo &IndexedAI : AllocSites) {
AllocationInfo AI;
AI.Info = IndexedAI.Info;
AI.CallStack = Callback(IndexedAI.CSId);
Record.AllocSites.push_back(std::move(AI));
}
Record.CallSites.reserve(CallSites.size());
for (const IndexedCallSiteInfo &CS : CallSites) {
std::vector<Frame> Frames = Callback(CS.CSId);
Record.CallSites.emplace_back(std::move(Frames), CS.CalleeGuids);
}
return Record;
}
GlobalValue::GUID getGUID(const StringRef FunctionName) {
// Canonicalize the function name to drop suffixes such as ".llvm.". Note
// we do not drop any ".__uniq." suffixes, as getCanonicalFnName does not drop
// those by default. This is by design to differentiate internal linkage
// functions during matching. By dropping the other suffixes we can then match
// functions in the profile use phase prior to their addition. Note that this
// applies to both instrumented and sampled function names.
StringRef CanonicalName =
sampleprof::FunctionSamples::getCanonicalFnName(FunctionName);
// We use the function guid which we expect to be a uint64_t. At
// this time, it is the lower 64 bits of the md5 of the canonical
// function name.
return Function::getGUIDAssumingExternalLinkage(CanonicalName);
}
Expected<MemProfSchema> readMemProfSchema(const unsigned char *&Buffer) {
using namespace support;
const unsigned char *Ptr = Buffer;
const uint64_t NumSchemaIds =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
if (NumSchemaIds > static_cast<uint64_t>(Meta::Size)) {
return make_error<InstrProfError>(instrprof_error::malformed,
"memprof schema invalid");
}
MemProfSchema Result;
for (size_t I = 0; I < NumSchemaIds; I++) {
const uint64_t Tag =
endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
if (Tag >= static_cast<uint64_t>(Meta::Size)) {
return make_error<InstrProfError>(instrprof_error::malformed,
"memprof schema invalid");
}
Result.push_back(static_cast<Meta>(Tag));
}
// Advance the buffer to one past the schema if we succeeded.
Buffer = Ptr;
return Result;
}
} // namespace memprof
} // namespace llvm
|