1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
/*
* Copyright (C) 2017-2020 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/os_interface/linux/drm_allocation.h"
#include "shared/source/os_interface/linux/drm_buffer_object.h"
#include "shared/source/os_interface/linux/drm_neo.h"
#include "drm_gem_close_worker.h"
#include <limits>
#include <map>
#include <sys/mman.h>
namespace NEO {
class BufferObject;
class Drm;
class DrmMemoryManager : public MemoryManager {
public:
DrmMemoryManager(gemCloseWorkerMode mode,
bool forcePinAllowed,
bool validateHostPtrMemory,
ExecutionEnvironment &executionEnvironment);
~DrmMemoryManager() override;
void initialize(gemCloseWorkerMode mode);
void addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) override;
void removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) override;
void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override;
void handleFenceCompletion(GraphicsAllocation *allocation) override;
GraphicsAllocation *createGraphicsAllocationFromExistingStorage(AllocationProperties &properties, void *ptr, MultiGraphicsAllocation &multiGraphicsAllocation) override;
GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness) override;
GraphicsAllocation *createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding) override;
GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex) override { return nullptr; }
uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override;
uint64_t getLocalMemorySize(uint32_t rootDeviceIndex) override;
AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
void cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
void commonCleanup() override;
// drm/i915 ioctl wrappers
MOCKABLE_VIRTUAL uint32_t unreference(BufferObject *bo, bool synchronousDestroy);
bool isValidateHostMemoryEnabled() const {
return validateHostPtrMemory;
}
DrmGemCloseWorker *peekGemCloseWorker() const { return this->gemCloseWorker.get(); }
bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy) override;
int obtainFdFromHandle(int boHandle, uint32_t rootDeviceindex);
AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override;
void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
MOCKABLE_VIRTUAL BufferObject *createBufferObjectInMemoryRegion(Drm *drm, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount);
std::unique_lock<std::mutex> acquireAllocLock();
std::vector<GraphicsAllocation *> &getSysMemAllocs();
std::vector<GraphicsAllocation *> &getLocalMemAllocs(uint32_t rootDeviceIndex);
void registerSysMemAlloc(GraphicsAllocation *allocation) override;
void registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex) override;
void unregisterAllocation(GraphicsAllocation *allocation);
protected:
BufferObject *findAndReferenceSharedBufferObject(int boHandle);
BufferObject *createSharedBufferObject(int boHandle, size_t size, bool requireSpecificBitness, uint32_t rootDeviceIndex);
void eraseSharedBufferObject(BufferObject *bo);
void pushSharedBufferObject(BufferObject *bo);
BufferObject *allocUserptr(uintptr_t address, size_t size, uint64_t flags, uint32_t rootDeviceIndex);
bool setDomainCpu(GraphicsAllocation &graphicsAllocation, bool writeEnable);
uint64_t acquireGpuRange(size_t &size, bool requireSpecificBitness, uint32_t rootDeviceIndex, bool requiresStandard64KBHeap);
MOCKABLE_VIRTUAL void releaseGpuRange(void *address, size_t size, uint32_t rootDeviceIndex);
void emitPinningRequest(BufferObject *bo, const AllocationData &allocationData) const;
uint32_t getDefaultDrmContextId() const;
size_t getUserptrAlignment();
DrmAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override;
DrmAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) override;
DrmAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override;
DrmAllocation *createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties);
DrmAllocation *createAllocWithAlignmentFromUserptr(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSVMSize, uint64_t gpuAddress);
DrmAllocation *createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress);
void obtainGpuAddress(const AllocationData &allocationData, BufferObject *bo, uint64_t gpuAddress);
DrmAllocation *allocateUSMHostGraphicsMemory(const AllocationData &allocationData) override;
DrmAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) override;
DrmAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override;
GraphicsAllocation *allocateShareableMemory(const AllocationData &allocationData) override;
GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override;
GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override;
void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
void *lockResourceInLocalMemoryImpl(GraphicsAllocation &graphicsAllocation);
MOCKABLE_VIRTUAL void *lockResourceInLocalMemoryImpl(BufferObject *bo);
MOCKABLE_VIRTUAL void unlockResourceInLocalMemoryImpl(BufferObject *bo);
void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
DrmAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData, bool useLocalMemory) override;
GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
bool createDrmAllocation(Drm *drm, DrmAllocation *allocation, uint64_t gpuAddress, size_t maxOsContextCount);
void registerAllocationInOs(GraphicsAllocation *allocation) override;
Drm &getDrm(uint32_t rootDeviceIndex) const;
uint32_t getRootDeviceIndex(const Drm *drm);
std::vector<BufferObject *> pinBBs;
std::vector<void *> memoryForPinBBs;
size_t pinThreshold = 8 * 1024 * 1024;
bool forcePinEnabled = false;
const bool validateHostPtrMemory;
std::unique_ptr<DrmGemCloseWorker> gemCloseWorker;
decltype(&mmap) mmapFunction = mmap;
decltype(&munmap) munmapFunction = munmap;
decltype(&lseek) lseekFunction = lseek;
decltype(&close) closeFunction = close;
std::vector<BufferObject *> sharingBufferObjects;
std::mutex mtx;
std::vector<std::vector<GraphicsAllocation *>> localMemAllocs;
std::vector<GraphicsAllocation *> sysMemAllocs;
std::mutex allocMutex;
};
} // namespace NEO
|
