1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
/*
* Copyright (C) 2020-2022 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
*/
#include "level_zero/core/source/event/event.h"
#include "shared/source/command_stream/command_stream_receiver_hw.h"
#include "shared/source/command_stream/csr_definitions.h"
#include "shared/source/debug_settings/debug_settings_manager.h"
#include "shared/source/device/device.h"
#include "shared/source/execution_environment/execution_environment.h"
#include "shared/source/execution_environment/root_device_environment.h"
#include "shared/source/helpers/constants.h"
#include "shared/source/helpers/string.h"
#include "shared/source/memory_manager/memory_manager.h"
#include "shared/source/memory_manager/memory_operations_handler.h"
#include "shared/source/utilities/cpuintrinsics.h"
#include "shared/source/utilities/wait_util.h"
#include "level_zero/core/source/cmdlist/cmdlist.h"
#include "level_zero/core/source/cmdqueue/cmdqueue.h"
#include "level_zero/core/source/context/context_imp.h"
#include "level_zero/core/source/device/device.h"
#include "level_zero/core/source/device/device_imp.h"
#include "level_zero/core/source/driver/driver_handle_imp.h"
#include "level_zero/core/source/hw_helpers/l0_hw_helper.h"
#include "level_zero/tools/source/metrics/metric.h"
#include <set>
//
#include "level_zero/core/source/event/event_impl.inl"
namespace L0 {
template Event *Event::create<uint64_t>(EventPool *, const ze_event_desc_t *, Device *);
template Event *Event::create<uint32_t>(EventPool *, const ze_event_desc_t *, Device *);
ze_result_t EventPoolImp::initialize(DriverHandle *driver, Context *context, uint32_t numDevices, ze_device_handle_t *phDevices) {
this->context = static_cast<ContextImp *>(context);
RootDeviceIndicesContainer rootDeviceIndices;
uint32_t maxRootDeviceIndex = 0u;
DriverHandleImp *driverHandleImp = static_cast<DriverHandleImp *>(driver);
bool useDevicesFromApi = true;
bool useDeviceAlloc = isEventPoolDeviceAllocationFlagSet();
if (numDevices == 0) {
numDevices = static_cast<uint32_t>(driverHandleImp->devices.size());
useDevicesFromApi = false;
}
for (uint32_t i = 0u; i < numDevices; i++) {
Device *eventDevice = nullptr;
if (useDevicesFromApi) {
eventDevice = Device::fromHandle(phDevices[i]);
} else {
eventDevice = driverHandleImp->devices[i];
}
if (!eventDevice) {
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}
devices.push_back(eventDevice);
rootDeviceIndices.push_back(eventDevice->getNEODevice()->getRootDeviceIndex());
if (maxRootDeviceIndex < eventDevice->getNEODevice()->getRootDeviceIndex()) {
maxRootDeviceIndex = eventDevice->getNEODevice()->getRootDeviceIndex();
}
}
rootDeviceIndices.remove_duplicates();
auto &hwHelper = devices[0]->getHwHelper();
useDeviceAlloc |= L0HwHelper::get(getDevice()->getHwInfo().platform.eRenderCoreFamily).alwaysAllocateEventInLocalMem();
eventAlignment = static_cast<uint32_t>(hwHelper.getTimestampPacketAllocatorAlignment());
eventSize = static_cast<uint32_t>(alignUp(EventPacketsCount::eventPackets * hwHelper.getSingleTimestampPacketSize(), eventAlignment));
size_t alignedSize = alignUp<size_t>(numEvents * eventSize, MemoryConstants::pageSize64k);
NEO::AllocationType allocationType = isEventPoolTimestampFlagSet() ? NEO::AllocationType::TIMESTAMP_PACKET_TAG_BUFFER
: NEO::AllocationType::BUFFER_HOST_MEMORY;
if (this->devices.size() > 1) {
useDeviceAlloc = false;
}
if (useDeviceAlloc) {
allocationType = NEO::AllocationType::GPU_TIMESTAMP_DEVICE_BUFFER;
}
eventPoolAllocations = std::make_unique<NEO::MultiGraphicsAllocation>(maxRootDeviceIndex);
bool allocatedMemory = false;
if (useDeviceAlloc) {
NEO::AllocationProperties allocationProperties{*rootDeviceIndices.begin(), alignedSize, allocationType, devices[0]->getNEODevice()->getDeviceBitfield()};
allocationProperties.alignment = eventAlignment;
auto memoryManager = driver->getMemoryManager();
auto graphicsAllocation = memoryManager->allocateGraphicsMemoryWithProperties(allocationProperties);
if (graphicsAllocation) {
eventPoolAllocations->addAllocation(graphicsAllocation);
allocatedMemory = true;
if (eventPoolFlags & ZE_EVENT_POOL_FLAG_IPC) {
uint64_t handle = 0;
this->isShareableEventMemory = (graphicsAllocation->peekInternalHandle(memoryManager, handle) == 0);
}
}
} else {
NEO::AllocationProperties allocationProperties{*rootDeviceIndices.begin(), alignedSize, allocationType, systemMemoryBitfield};
allocationProperties.alignment = eventAlignment;
eventPoolPtr = driver->getMemoryManager()->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices,
allocationProperties,
*eventPoolAllocations);
if (eventPoolFlags & ZE_EVENT_POOL_FLAG_IPC) {
this->isShareableEventMemory = eventPoolAllocations->getDefaultGraphicsAllocation()->isShareableHostMemory;
}
allocatedMemory = (nullptr != eventPoolPtr);
}
if (!allocatedMemory) {
return ZE_RESULT_ERROR_OUT_OF_DEVICE_MEMORY;
}
return ZE_RESULT_SUCCESS;
}
EventPoolImp::~EventPoolImp() {
if (eventPoolAllocations) {
auto graphicsAllocations = eventPoolAllocations->getGraphicsAllocations();
auto memoryManager = devices[0]->getDriverHandle()->getMemoryManager();
for (auto gpuAllocation : graphicsAllocations) {
memoryManager->freeGraphicsMemory(gpuAllocation);
}
}
}
ze_result_t EventPoolImp::destroy() {
delete this;
return ZE_RESULT_SUCCESS;
}
ze_result_t EventPoolImp::createEvent(const ze_event_desc_t *desc, ze_event_handle_t *phEvent) {
if (desc->index > (getNumEvents() - 1)) {
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}
auto &l0HwHelper = L0HwHelper::get(getDevice()->getHwInfo().platform.eRenderCoreFamily);
*phEvent = l0HwHelper.createEvent(this, desc, getDevice());
return ZE_RESULT_SUCCESS;
}
ze_result_t Event::destroy() {
delete this;
return ZE_RESULT_SUCCESS;
}
EventPool *EventPool::create(DriverHandle *driver, Context *context, uint32_t numDevices, ze_device_handle_t *phDevices, const ze_event_pool_desc_t *desc, ze_result_t &result) {
auto eventPool = std::make_unique<EventPoolImp>(desc);
if (!eventPool) {
result = ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
DEBUG_BREAK_IF(true);
return nullptr;
}
result = eventPool->initialize(driver, context, numDevices, phDevices);
if (result) {
return nullptr;
}
return eventPool.release();
}
} // namespace L0
|
