1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
/*
* Copyright (C) 2019-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/helpers/common_types.h"
#include "shared/source/memory_manager/multi_graphics_allocation.h"
#include "shared/source/memory_manager/residency_container.h"
#include "shared/source/unified_memory/unified_memory.h"
#include "memory_properties_flags.h"
#include <atomic>
#include <cstdint>
#include <map>
#include <mutex>
#include <set>
#include <shared_mutex>
namespace NEO {
class CommandStreamReceiver;
class GraphicsAllocation;
class MemoryManager;
class Device;
struct SvmAllocationData {
SvmAllocationData(uint32_t maxRootDeviceIndex) : gpuAllocations(maxRootDeviceIndex), maxRootDeviceIndex(maxRootDeviceIndex){};
SvmAllocationData(const SvmAllocationData &svmAllocData) : SvmAllocationData(svmAllocData.maxRootDeviceIndex) {
this->allocationFlagsProperty = svmAllocData.allocationFlagsProperty;
this->cpuAllocation = svmAllocData.cpuAllocation;
this->device = svmAllocData.device;
this->size = svmAllocData.size;
this->memoryType = svmAllocData.memoryType;
this->allocId = svmAllocData.allocId;
this->pageSizeForAlignment = svmAllocData.pageSizeForAlignment;
this->isImportedAllocation = svmAllocData.isImportedAllocation;
for (auto allocation : svmAllocData.gpuAllocations.getGraphicsAllocations()) {
if (allocation) {
this->gpuAllocations.addAllocation(allocation);
}
}
}
SvmAllocationData &operator=(const SvmAllocationData &) = delete;
GraphicsAllocation *cpuAllocation = nullptr;
MultiGraphicsAllocation gpuAllocations;
size_t size = 0;
size_t pageSizeForAlignment = 0;
InternalMemoryType memoryType = InternalMemoryType::SVM;
MemoryProperties allocationFlagsProperty;
Device *device = nullptr;
bool isImportedAllocation = false;
void setAllocId(uint32_t id) {
allocId = id;
}
uint32_t getAllocId() {
return allocId;
}
static constexpr uint32_t uninitializedAllocId = std::numeric_limits<uint32_t>::max();
protected:
const uint32_t maxRootDeviceIndex;
uint32_t allocId = uninitializedAllocId;
};
struct SvmMapOperation {
void *regionSvmPtr = nullptr;
size_t regionSize = 0;
void *baseSvmPtr = nullptr;
size_t offset = 0;
bool readOnlyMap = false;
};
class SVMAllocsManager {
public:
class MapBasedAllocationTracker {
friend class SVMAllocsManager;
public:
using SvmAllocationContainer = std::map<const void *, SvmAllocationData>;
void insert(SvmAllocationData);
void remove(SvmAllocationData);
SvmAllocationData *get(const void *);
size_t getNumAllocs() const { return allocations.size(); };
SvmAllocationContainer allocations;
};
struct MapOperationsTracker {
using SvmMapOperationsContainer = std::map<const void *, SvmMapOperation>;
void insert(SvmMapOperation);
void remove(const void *);
SvmMapOperation *get(const void *);
size_t getNumMapOperations() const { return operations.size(); };
protected:
SvmMapOperationsContainer operations;
};
struct SvmAllocationProperties {
bool coherent = false;
bool hostPtrReadOnly = false;
bool readOnly = false;
};
struct InternalAllocationsTracker {
uint32_t latestSentTaskCount = 0lu;
uint32_t latestResidentObjectId = 0lu;
};
struct UnifiedMemoryProperties {
UnifiedMemoryProperties(InternalMemoryType memoryType,
const RootDeviceIndicesContainer &rootDeviceIndices,
const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields) : memoryType(memoryType),
rootDeviceIndices(rootDeviceIndices),
subdeviceBitfields(subdeviceBitfields){};
InternalMemoryType memoryType = InternalMemoryType::NOT_SPECIFIED;
MemoryProperties allocationFlags;
Device *device = nullptr;
const RootDeviceIndicesContainer &rootDeviceIndices;
const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields;
};
struct SvmCacheAllocationInfo {
size_t allocationSize;
void *allocation;
SvmCacheAllocationInfo(size_t allocationSize, void *allocation) : allocationSize(allocationSize), allocation(allocation) {}
bool operator<(SvmCacheAllocationInfo const &other) {
return allocationSize < other.allocationSize;
}
bool operator<(size_t const &size) {
return allocationSize < size;
}
};
struct SvmAllocationCache {
void insert(size_t size, void *);
void *get(size_t size, const UnifiedMemoryProperties &unifiedMemoryProperties, SVMAllocsManager *svmAllocsManager);
void trim(SVMAllocsManager *svmAllocsManager);
std::vector<SvmCacheAllocationInfo> allocations;
std::mutex mtx;
};
SVMAllocsManager(MemoryManager *memoryManager, bool multiOsContextSupport);
MOCKABLE_VIRTUAL ~SVMAllocsManager();
void *createSVMAlloc(size_t size,
const SvmAllocationProperties svmProperties,
const RootDeviceIndicesContainer &rootDeviceIndices,
const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields);
MOCKABLE_VIRTUAL void *createHostUnifiedMemoryAllocation(size_t size,
const UnifiedMemoryProperties &svmProperties);
MOCKABLE_VIRTUAL void *createUnifiedMemoryAllocation(size_t size,
const UnifiedMemoryProperties &svmProperties);
MOCKABLE_VIRTUAL void *createSharedUnifiedMemoryAllocation(size_t size,
const UnifiedMemoryProperties &svmProperties,
void *cmdQ);
void *createUnifiedKmdMigratedAllocation(size_t size,
const SvmAllocationProperties &svmProperties,
const UnifiedMemoryProperties &unifiedMemoryProperties);
void setUnifiedAllocationProperties(GraphicsAllocation *allocation, const SvmAllocationProperties &svmProperties);
SvmAllocationData *getSVMAlloc(const void *ptr);
MOCKABLE_VIRTUAL bool freeSVMAlloc(void *ptr, bool blocking);
MOCKABLE_VIRTUAL void freeSVMAllocImpl(void *ptr, bool blocking, SvmAllocationData *svmData);
bool freeSVMAlloc(void *ptr) { return freeSVMAlloc(ptr, false); }
void trimUSMDeviceAllocCache();
void insertSVMAlloc(const SvmAllocationData &svmData);
void removeSVMAlloc(const SvmAllocationData &svmData);
size_t getNumAllocs() const { return SVMAllocs.getNumAllocs(); }
MapBasedAllocationTracker *getSVMAllocs() { return &SVMAllocs; }
MOCKABLE_VIRTUAL void insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap);
void removeSvmMapOperation(const void *regionSvmPtr);
SvmMapOperation *getSvmMapOperation(const void *regionPtr);
MOCKABLE_VIRTUAL void addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,
ResidencyContainer &residencyContainer,
uint32_t requestedTypesMask);
void makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t requestedTypesMask);
void *createUnifiedAllocationWithDeviceStorage(size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties);
void freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData);
bool hasHostAllocations();
std::atomic<uint32_t> allocationsCounter = 0;
MOCKABLE_VIRTUAL void makeIndirectAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t taskCount);
void prepareIndirectAllocationForDestruction(SvmAllocationData *);
void prefetchMemory(Device &device, SvmAllocationData &svmData);
std::unique_lock<std::mutex> obtainOwnership();
std::map<CommandStreamReceiver *, InternalAllocationsTracker> indirectAllocationsResidency;
using NonGpuDomainAllocsContainer = std::vector<void *>;
NonGpuDomainAllocsContainer nonGpuDomainAllocs;
protected:
void *createZeroCopySvmAllocation(size_t size, const SvmAllocationProperties &svmProperties,
const RootDeviceIndicesContainer &rootDeviceIndices,
const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields);
AllocationType getGraphicsAllocationTypeAndCompressionPreference(const UnifiedMemoryProperties &unifiedMemoryProperties, bool &compressionEnabled) const;
void freeZeroCopySvmAllocation(SvmAllocationData *svmData);
void initUsmDeviceAllocationsCache();
MapBasedAllocationTracker SVMAllocs;
MapOperationsTracker svmMapOperations;
MemoryManager *memoryManager;
std::shared_mutex mtx;
std::mutex mtxForIndirectAccess;
bool multiOsContextSupport;
SvmAllocationCache usmDeviceAllocationsCache;
bool usmDeviceAllocationsCacheEnabled = false;
};
} // namespace NEO
|
