1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
|
/*
* Copyright (C) 2018-2025 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#pragma once
#include "shared/source/command_stream/task_count_helper.h"
#include "shared/source/helpers/common_types.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/non_copyable_or_moveable.h"
#include "shared/source/memory_manager/definitions/engine_limits.h"
#include "shared/source/memory_manager/memory_operations_status.h"
#include "shared/source/os_interface/linux/cache_info.h"
#include "shared/source/utilities/stackvec.h"
#include <array>
#include <atomic>
#include <cstddef>
#include <mutex>
#include <stdint.h>
#include <utility>
#include <vector>
namespace NEO {
struct ExecBuffer;
struct ExecObject;
class DrmMemoryManager;
class Drm;
class OsContext;
class BufferObjectHandleWrapper : NEO::NonCopyableClass {
private:
struct ControlBlock {
int refCount{0};
int weakRefCount{0};
std::mutex blockMutex{};
};
enum class Ownership : std::uint8_t {
weak = 0,
strong = 1,
};
public:
explicit BufferObjectHandleWrapper(int boHandle, uint32_t rootDeviceIndex) noexcept
: boHandle{boHandle}, rootDeviceIndex(rootDeviceIndex) {}
BufferObjectHandleWrapper(BufferObjectHandleWrapper &&other) noexcept
: boHandle(std::exchange(other.boHandle, -1)), rootDeviceIndex(std::exchange(other.rootDeviceIndex, UINT32_MAX)), ownership(other.ownership), controlBlock(std::exchange(other.controlBlock, nullptr)) {}
~BufferObjectHandleWrapper();
BufferObjectHandleWrapper &operator=(BufferObjectHandleWrapper &&) = delete;
BufferObjectHandleWrapper acquireSharedOwnership();
BufferObjectHandleWrapper acquireWeakOwnership();
bool canCloseBoHandle();
int getBoHandle() const {
return boHandle;
}
uint32_t getRootDeviceIndex() const {
return rootDeviceIndex;
}
void setBoHandle(int handle) {
boHandle = handle;
}
void setRootDeviceIndex(uint32_t index) {
rootDeviceIndex = index;
}
protected:
BufferObjectHandleWrapper(int boHandle, uint32_t rootDeviceIndex, Ownership ownership, ControlBlock *controlBlock)
: boHandle{boHandle}, rootDeviceIndex{rootDeviceIndex}, ownership{ownership}, controlBlock{controlBlock} {}
int boHandle{};
uint32_t rootDeviceIndex{UINT32_MAX};
Ownership ownership{Ownership::strong};
ControlBlock *controlBlock{nullptr};
};
static_assert(NEO::NonCopyable<BufferObjectHandleWrapper>);
class BufferObject {
public:
BufferObject(uint32_t rootDeviceIndex, Drm *drm, uint64_t patIndex, int handle, size_t size, size_t maxOsContextCount);
BufferObject(uint32_t rootDeviceIndex, Drm *drm, uint64_t patIndex, BufferObjectHandleWrapper &&handle, size_t size, size_t maxOsContextCount);
MOCKABLE_VIRTUAL ~BufferObject() = default;
struct Deleter {
void operator()(BufferObject *bo) {
bo->close();
delete bo;
}
};
enum class BOType {
legacy,
coherent,
nonCoherent
};
bool setTiling(uint32_t mode, uint32_t stride);
int pin(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId);
MOCKABLE_VIRTUAL int validateHostPtr(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId);
MOCKABLE_VIRTUAL int exec(uint32_t used, size_t startOffset, unsigned int flags, bool requiresCoherency, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId,
BufferObject *const residency[], size_t residencyCount, ExecObject *execObjectsStorage, uint64_t completionGpuAddress, TaskCountType completionValue);
int bind(OsContext *osContext, uint32_t vmHandleId, const bool forcePagingFence);
MOCKABLE_VIRTUAL int unbind(OsContext *osContext, uint32_t vmHandleId);
void printExecutionBuffer(ExecBuffer &execbuf, const size_t &residencyCount, ExecObject *execObjectsStorage, BufferObject *const residency[]);
int wait(int64_t timeoutNs);
bool close();
inline void reference() {
this->refCount++;
}
inline uint32_t unreference() {
return this->refCount.fetch_sub(1);
}
uint32_t getRefCount() const;
bool isBoHandleShared() const {
return boHandleShared;
}
void markAsSharedBoHandle() {
boHandleShared = true;
}
BufferObjectHandleWrapper acquireSharedOwnershipOfBoHandle() {
markAsSharedBoHandle();
return handle.acquireSharedOwnership();
}
BufferObjectHandleWrapper acquireWeakOwnershipOfBoHandle() {
markAsSharedBoHandle();
return handle.acquireWeakOwnership();
}
size_t peekSize() const { return size; }
int peekHandle() const { return handle.getBoHandle(); }
const Drm *peekDrm() const { return drm; }
uint64_t peekAddress() const { return gpuAddress; }
void setAddress(uint64_t address);
void *peekLockedAddress() const { return lockedAddress; }
void setLockedAddress(void *cpuAddress) { this->lockedAddress = cpuAddress; }
void setUnmapSize(uint64_t unmapSize) { this->unmapSize = unmapSize; }
uint64_t peekUnmapSize() const { return unmapSize; }
bool peekIsReusableAllocation() const { return this->isReused; }
void markAsReusableAllocation() { this->isReused = true; }
void addBindExtHandle(uint32_t handle);
const StackVec<uint32_t, 2> &getBindExtHandles() const { return bindExtHandles; }
void markForCapture() {
allowCapture = true;
}
bool isMarkedForCapture() {
return allowCapture;
}
bool isImmediateBindingRequired() {
return requiresImmediateBinding;
}
bool isReadOnlyGpuResource() {
return readOnlyGpuResource;
}
void setAsReadOnly(bool isReadOnly) {
readOnlyGpuResource = isReadOnly;
}
void requireImmediateBinding(bool required) {
requiresImmediateBinding = required;
}
bool isExplicitResidencyRequired() {
return requiresExplicitResidency;
}
void requireExplicitResidency(bool required) {
requiresExplicitResidency = required;
}
uint32_t getRootDeviceIndex() {
return rootDeviceIndex;
}
int getHandle() {
return handle.getBoHandle();
}
void setCacheRegion(CacheRegion regionIndex) { cacheRegion = regionIndex; }
CacheRegion peekCacheRegion() const { return cacheRegion; }
void setCachePolicy(CachePolicy memType) { cachePolicy = memType; }
CachePolicy peekCachePolicy() const { return cachePolicy; }
void setColourWithBind() {
this->colourWithBind = true;
}
void setColourChunk(size_t size) {
this->colourChunk = size;
}
void addColouringAddress(uint64_t address) {
this->bindAddresses.push_back(address);
}
void reserveAddressVector(size_t size) {
this->bindAddresses.reserve(size);
}
bool getColourWithBind() {
return this->colourWithBind;
}
size_t getColourChunk() {
return this->colourChunk;
}
std::vector<uint64_t> &getColourAddresses() {
return this->bindAddresses;
}
void requireExplicitLockedMemory(bool locked) { requiresLocked = locked; }
bool isExplicitLockedMemoryRequired() { return requiresLocked; }
uint64_t peekPatIndex() const { return patIndex; }
void setPatIndex(uint64_t newPatIndex) { this->patIndex = newPatIndex; }
BOType peekBOType() const { return boType; }
void setBOType(BOType newBoType) { this->boType = newBoType; }
void setUserptr(uint64_t ptr) { this->userptr = ptr; };
uint64_t getUserptr() const { return this->userptr; };
void setMmapOffset(uint64_t offset) { this->mmapOffset = offset; };
uint64_t getMmapOffset() const { return this->mmapOffset; };
static constexpr int gpuHangDetected{-7171};
uint32_t getOsContextId(OsContext *osContext);
const auto &getBindInfo() const { return bindInfo; }
void setChunked(bool chunked) { this->chunked = chunked; }
bool isChunked() const { return this->chunked; }
void setRegisteredBindHandleCookie(uint64_t cookie) {
registeredBindHandleCookie = cookie;
}
uint64_t getRegisteredBindHandleCookie() {
return registeredBindHandleCookie;
}
protected:
MOCKABLE_VIRTUAL MemoryOperationsStatus evictUnusedAllocations(bool waitForCompletion, bool isLockNeeded);
MOCKABLE_VIRTUAL void fillExecObject(ExecObject &execObject, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId);
void printBOBindingResult(OsContext *osContext, uint32_t vmHandleId, bool bind, int retVal);
Drm *drm = nullptr;
BufferObjectHandleWrapper handle; // i915 gem object handle
void *lockedAddress = nullptr; // CPU side virtual address
uint64_t size = 0;
uint64_t unmapSize = 0;
uint64_t patIndex = CommonConstants::unsupportedPatIndex;
uint64_t userptr = 0u;
uint64_t mmapOffset = 0u;
size_t colourChunk = 0;
uint64_t gpuAddress = 0llu;
std::vector<uint64_t> bindAddresses;
std::vector<std::array<bool, EngineLimits::maxHandleCount>> bindInfo;
StackVec<uint32_t, 2> bindExtHandles;
uint64_t registeredBindHandleCookie = 0;
BOType boType = BOType::legacy;
std::atomic<uint32_t> refCount;
uint32_t rootDeviceIndex = std::numeric_limits<uint32_t>::max();
uint32_t tilingMode = 0;
CachePolicy cachePolicy = CachePolicy::writeBack;
CacheRegion cacheRegion = CacheRegion::defaultRegion;
bool colourWithBind = false;
bool perContextVmsUsed = false;
bool boHandleShared = false;
bool allowCapture = false;
bool requiresImmediateBinding = false;
bool requiresExplicitResidency = false;
bool requiresLocked = false;
bool chunked = false;
bool isReused = false;
bool readOnlyGpuResource = false;
};
} // namespace NEO
|