1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/command_stream/preemption_mode.h"
#include "shared/source/helpers/aligned_memory.h"
#include "shared/source/helpers/bit_helpers.h"
#include "shared/source/helpers/common_types.h"
#include "shared/source/helpers/engine_control.h"
#include "shared/source/helpers/heap_assigner.h"
#include "shared/source/helpers/hw_helper.h"
#include "shared/source/memory_manager/allocation_properties.h"
#include "shared/source/memory_manager/gfx_partition.h"
#include "shared/source/memory_manager/graphics_allocation.h"
#include "shared/source/memory_manager/host_ptr_defines.h"
#include "shared/source/memory_manager/local_memory_usage.h"
#include "shared/source/memory_manager/multi_graphics_allocation.h"
#include "shared/source/page_fault_manager/cpu_page_fault_manager.h"
#include "engine_node.h"
#include <bitset>
#include <cstdint>
#include <mutex>
#include <vector>
namespace NEO {
class DeferredDeleter;
class ExecutionEnvironment;
class Gmm;
class HostPtrManager;
class OsContext;
enum AllocationUsage {
TEMPORARY_ALLOCATION,
REUSABLE_ALLOCATION
};
struct AlignedMallocRestrictions {
uintptr_t minAddress;
};
struct AddressRange {
uint64_t address;
size_t size;
};
constexpr size_t paddingBufferSize = 2 * MemoryConstants::megaByte;
namespace MemoryTransferHelper {
bool transferMemoryToAllocation(bool useBlitter, const Device &device, GraphicsAllocation *dstAllocation, size_t dstOffset, const void *srcMemory, size_t srcSize);
}
class MemoryManager {
public:
enum AllocationStatus {
Success = 0,
Error,
InvalidHostPointer,
RetryInNonDevicePool
};
MemoryManager(ExecutionEnvironment &executionEnvironment);
bool isInitialized() const { return initialized; }
virtual ~MemoryManager();
MOCKABLE_VIRTUAL void *allocateSystemMemory(size_t size, size_t alignment);
virtual void addAllocationToHostPtrManager(GraphicsAllocation *memory) = 0;
virtual void removeAllocationFromHostPtrManager(GraphicsAllocation *memory) = 0;
MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties) {
return allocateGraphicsMemoryInPreferredPool(properties, nullptr);
}
MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryWithProperties(const AllocationProperties &properties, const void *ptr) {
return allocateGraphicsMemoryInPreferredPool(properties, ptr);
}
GraphicsAllocation *allocateInternalGraphicsMemoryWithHostCopy(uint32_t rootDeviceIndex, DeviceBitfield bitField, const void *ptr, size_t size);
MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryInPreferredPool(const AllocationProperties &properties, const void *hostPtr);
virtual bool verifyHandle(osHandle handle, uint32_t rootDeviceIndex, bool) { return true; }
virtual GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) = 0;
virtual GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) = 0;
virtual bool mapAuxGpuVA(GraphicsAllocation *graphicsAllocation);
void *lockResource(GraphicsAllocation *graphicsAllocation);
void unlockResource(GraphicsAllocation *graphicsAllocation);
void cleanGraphicsMemoryCreatedFromHostPtr(GraphicsAllocation *);
GraphicsAllocation *createGraphicsAllocationWithPadding(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding);
virtual GraphicsAllocation *createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding);
MOCKABLE_VIRTUAL void *createMultiGraphicsAllocation(std::vector<uint32_t> &rootDeviceIndices, AllocationProperties &properties, MultiGraphicsAllocation &multiGraphicsAllocation);
virtual GraphicsAllocation *createGraphicsAllocationFromExistingStorage(AllocationProperties &properties, void *ptr, MultiGraphicsAllocation &multiGraphicsAllocation);
virtual AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) = 0;
virtual void cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) = 0;
void freeSystemMemory(void *ptr);
virtual void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) = 0;
MOCKABLE_VIRTUAL void freeGraphicsMemory(GraphicsAllocation *gfxAllocation);
virtual void handleFenceCompletion(GraphicsAllocation *allocation){};
void checkGpuUsageAndDestroyGraphicsAllocations(GraphicsAllocation *gfxAllocation);
virtual uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) = 0;
virtual uint64_t getLocalMemorySize(uint32_t rootDeviceIndex) = 0;
uint64_t getMaxApplicationAddress() { return is64bit ? MemoryConstants::max64BitAppAddress : MemoryConstants::max32BitAppAddress; };
MOCKABLE_VIRTUAL uint64_t getInternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory) { return getGfxPartition(rootDeviceIndex)->getHeapBase(selectInternalHeap(useLocalMemory)); }
uint64_t getExternalHeapBaseAddress(uint32_t rootDeviceIndex, bool useLocalMemory) { return getGfxPartition(rootDeviceIndex)->getHeapBase(selectExternalHeap(useLocalMemory)); }
bool isLimitedRange(uint32_t rootDeviceIndex) { return getGfxPartition(rootDeviceIndex)->isLimitedRange(); }
bool peek64kbPagesEnabled(uint32_t rootDeviceIndex) const;
bool peekForce32BitAllocations() const { return force32bitAllocations; }
void setForce32BitAllocations(bool newValue) { force32bitAllocations = newValue; }
bool peekVirtualPaddingSupport() const { return virtualPaddingAvailable; }
void setVirtualPaddingSupport(bool virtualPaddingSupport) { virtualPaddingAvailable = virtualPaddingSupport; }
DeferredDeleter *getDeferredDeleter() const {
return deferredDeleter.get();
}
PageFaultManager *getPageFaultManager() const {
return pageFaultManager.get();
}
void waitForDeletions();
void waitForEnginesCompletion(GraphicsAllocation &graphicsAllocation);
void cleanTemporaryAllocationListOnAllEngines(bool waitForCompletion);
bool isAsyncDeleterEnabled() const;
bool isLocalMemorySupported(uint32_t rootDeviceIndex) const;
virtual bool isMemoryBudgetExhausted() const;
virtual AlignedMallocRestrictions *getAlignedMallocRestrictions() {
return nullptr;
}
MOCKABLE_VIRTUAL void *alignedMallocWrapper(size_t bytes, size_t alignment) {
return ::alignedMalloc(bytes, alignment);
}
MOCKABLE_VIRTUAL void alignedFreeWrapper(void *ptr) {
::alignedFree(ptr);
}
MOCKABLE_VIRTUAL bool isHostPointerTrackingEnabled(uint32_t rootDeviceIndex);
void setForceNonSvmForExternalHostPtr(bool mode) {
forceNonSvmForExternalHostPtr = mode;
}
const ExecutionEnvironment &peekExecutionEnvironment() const { return executionEnvironment; }
OsContext *createAndRegisterOsContext(CommandStreamReceiver *commandStreamReceiver, aub_stream::EngineType engineType,
DeviceBitfield deviceBitfield, PreemptionMode preemptionMode,
bool lowPriority, bool internalEngine, bool rootDevice);
uint32_t getRegisteredEnginesCount() const { return static_cast<uint32_t>(registeredEngines.size()); }
EngineControlContainer &getRegisteredEngines();
EngineControl *getRegisteredEngineForCsr(CommandStreamReceiver *commandStreamReceiver);
void unregisterEngineForCsr(CommandStreamReceiver *commandStreamReceiver);
HostPtrManager *getHostPtrManager() const { return hostPtrManager.get(); }
void setDefaultEngineIndex(uint32_t index) { defaultEngineIndex = index; }
virtual bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy);
HeapIndex selectHeap(const GraphicsAllocation *allocation, bool hasPointer, bool isFullRangeSVM, bool useFrontWindow);
static std::unique_ptr<MemoryManager> createMemoryManager(ExecutionEnvironment &executionEnvironment);
virtual void *reserveCpuAddressRange(size_t size, uint32_t rootDeviceIndex) { return nullptr; };
virtual void releaseReservedCpuAddressRange(void *reserved, size_t size, uint32_t rootDeviceIndex){};
void *getReservedMemory(size_t size, size_t alignment);
GfxPartition *getGfxPartition(uint32_t rootDeviceIndex) { return gfxPartitions.at(rootDeviceIndex).get(); }
virtual AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) = 0;
virtual void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) = 0;
static HeapIndex selectInternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_INTERNAL; }
static HeapIndex selectExternalHeap(bool useLocalMemory) { return useLocalMemory ? HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY : HeapIndex::HEAP_EXTERNAL; }
static uint32_t maxOsContextCount;
virtual void commonCleanup(){};
virtual bool isCpuCopyRequired(const void *ptr) { return false; }
virtual void registerSysMemAlloc(GraphicsAllocation *allocation){};
virtual void registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex){};
protected:
bool getAllocationData(AllocationData &allocationData, const AllocationProperties &properties, const void *hostPtr, const StorageInfo &storageInfo);
static void overrideAllocationData(AllocationData &allocationData, const AllocationProperties &properties);
static bool isCopyRequired(ImageInfo &imgInfo, const void *hostPtr);
bool useNonSvmHostPtrAlloc(GraphicsAllocation::AllocationType allocationType, uint32_t rootDeviceIndex);
StorageInfo createStorageInfoFromProperties(const AllocationProperties &properties);
virtual GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) = 0;
virtual GraphicsAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) = 0;
GraphicsAllocation *allocateGraphicsMemory(const AllocationData &allocationData);
virtual GraphicsAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData);
virtual GraphicsAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) = 0;
virtual GraphicsAllocation *allocateUSMHostGraphicsMemory(const AllocationData &allocationData) = 0;
virtual GraphicsAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) = 0;
virtual GraphicsAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData, bool useLocalMemory) = 0;
virtual GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) = 0;
virtual GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) = 0;
GraphicsAllocation *allocateGraphicsMemoryForImageFromHostPtr(const AllocationData &allocationData);
MOCKABLE_VIRTUAL GraphicsAllocation *allocateGraphicsMemoryForImage(const AllocationData &allocationData);
virtual GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) = 0;
virtual GraphicsAllocation *allocateShareableMemory(const AllocationData &allocationData) = 0;
virtual void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) = 0;
virtual void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) = 0;
virtual void freeAssociatedResourceImpl(GraphicsAllocation &graphicsAllocation) { return unlockResourceImpl(graphicsAllocation); };
virtual void registerAllocationInOs(GraphicsAllocation *allocation) {}
bool initialized = false;
bool forceNonSvmForExternalHostPtr = false;
bool force32bitAllocations = false;
bool virtualPaddingAvailable = false;
std::unique_ptr<DeferredDeleter> deferredDeleter;
bool asyncDeleterEnabled = false;
std::vector<bool> enable64kbpages;
std::vector<bool> localMemorySupported;
bool supportsMultiStorageResources = true;
ExecutionEnvironment &executionEnvironment;
EngineControlContainer registeredEngines;
std::unique_ptr<HostPtrManager> hostPtrManager;
uint32_t latestContextId = std::numeric_limits<uint32_t>::max();
uint32_t defaultEngineIndex = 0;
std::unique_ptr<DeferredDeleter> multiContextResourceDestructor;
std::vector<std::unique_ptr<GfxPartition>> gfxPartitions;
std::vector<std::unique_ptr<LocalMemoryUsageBankSelector>> localMemoryUsageBankSelector;
void *reservedMemory = nullptr;
std::unique_ptr<PageFaultManager> pageFaultManager;
OSMemory::ReservedCpuAddressRange reservedCpuAddressRange;
HeapAssigner heapAssigner;
};
std::unique_ptr<DeferredDeleter> createDeferredDeleter();
} // namespace NEO
|
