1 /*
2 * Copyright (C) 2019-2021 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8 #include "shared/source/command_container/cmdcontainer.h"
9
10 #include "shared/source/command_container/command_encoder.h"
11 #include "shared/source/command_stream/command_stream_receiver.h"
12 #include "shared/source/command_stream/linear_stream.h"
13 #include "shared/source/device/device.h"
14 #include "shared/source/helpers/api_specific_config.h"
15 #include "shared/source/helpers/debug_helpers.h"
16 #include "shared/source/helpers/heap_helper.h"
17 #include "shared/source/helpers/hw_helper.h"
18 #include "shared/source/indirect_heap/indirect_heap.h"
19 #include "shared/source/memory_manager/allocations_list.h"
20 #include "shared/source/memory_manager/memory_manager.h"
21
22 namespace NEO {
23
~CommandContainer()24 CommandContainer::~CommandContainer() {
25 if (!device) {
26 DEBUG_BREAK_IF(device);
27 return;
28 }
29
30 this->handleCmdBufferAllocations(0u);
31
32 for (auto allocationIndirectHeap : allocationIndirectHeaps) {
33 if (heapHelper) {
34 heapHelper->storeHeapAllocation(allocationIndirectHeap);
35 }
36 }
37 for (auto deallocation : deallocationContainer) {
38 if (((deallocation->getAllocationType() == GraphicsAllocation::AllocationType::INTERNAL_HEAP) || (deallocation->getAllocationType() == GraphicsAllocation::AllocationType::LINEAR_STREAM))) {
39 getHeapHelper()->storeHeapAllocation(deallocation);
40 }
41 }
42 }
43
initialize(Device * device,AllocationsList * reusableAllocationList)44 ErrorCode CommandContainer::initialize(Device *device, AllocationsList *reusableAllocationList) {
45 this->device = device;
46 this->reusableAllocationList = reusableAllocationList;
47
48 size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);
49
50 auto cmdBufferAllocation = this->obtainNextCommandBufferAllocation();
51
52 if (!cmdBufferAllocation) {
53 return ErrorCode::OUT_OF_DEVICE_MEMORY;
54 }
55
56 cmdBufferAllocations.push_back(cmdBufferAllocation);
57
58 commandStream = std::unique_ptr<LinearStream>(new LinearStream(cmdBufferAllocation->getUnderlyingBuffer(),
59 defaultListCmdBufferSize));
60 commandStream->replaceGraphicsAllocation(cmdBufferAllocation);
61
62 if (!getFlushTaskUsedForImmediate()) {
63 addToResidencyContainer(cmdBufferAllocation);
64 }
65
66 constexpr size_t heapSize = 65536u;
67 heapHelper = std::unique_ptr<HeapHelper>(new HeapHelper(device->getMemoryManager(), device->getDefaultEngine().commandStreamReceiver->getInternalAllocationStorage(), device->getNumGenericSubDevices() > 1u));
68
69 for (uint32_t i = 0; i < IndirectHeap::Type::NUM_TYPES; i++) {
70 if (NEO::ApiSpecificConfig::getBindlessConfiguration() && i != IndirectHeap::INDIRECT_OBJECT) {
71 continue;
72 }
73 allocationIndirectHeaps[i] = heapHelper->getHeapAllocation(i,
74 heapSize,
75 alignedSize,
76 device->getRootDeviceIndex());
77 if (!allocationIndirectHeaps[i]) {
78 return ErrorCode::OUT_OF_DEVICE_MEMORY;
79 }
80 residencyContainer.push_back(allocationIndirectHeaps[i]);
81
82 bool requireInternalHeap = (IndirectHeap::INDIRECT_OBJECT == i);
83 indirectHeaps[i] = std::make_unique<IndirectHeap>(allocationIndirectHeaps[i], requireInternalHeap);
84 if (i == IndirectHeap::Type::SURFACE_STATE) {
85 indirectHeaps[i]->getSpace(reservedSshSize);
86 }
87 }
88
89 indirectObjectHeapBaseAddress = device->getMemoryManager()->getInternalHeapBaseAddress(device->getRootDeviceIndex(), allocationIndirectHeaps[IndirectHeap::Type::INDIRECT_OBJECT]->isAllocatedInLocalMemoryPool());
90
91 instructionHeapBaseAddress = device->getMemoryManager()->getInternalHeapBaseAddress(device->getRootDeviceIndex(), device->getMemoryManager()->isLocalMemoryUsedForIsa(device->getRootDeviceIndex()));
92
93 iddBlock = nullptr;
94 nextIddInBlock = this->getNumIddPerBlock();
95
96 return ErrorCode::SUCCESS;
97 }
98
addToResidencyContainer(GraphicsAllocation * alloc)99 void CommandContainer::addToResidencyContainer(GraphicsAllocation *alloc) {
100 if (alloc == nullptr) {
101 return;
102 }
103
104 this->residencyContainer.push_back(alloc);
105 }
106
removeDuplicatesFromResidencyContainer()107 void CommandContainer::removeDuplicatesFromResidencyContainer() {
108 std::sort(this->residencyContainer.begin(), this->residencyContainer.end());
109 this->residencyContainer.erase(std::unique(this->residencyContainer.begin(), this->residencyContainer.end()), this->residencyContainer.end());
110 }
111
reset()112 void CommandContainer::reset() {
113 setDirtyStateForAllHeaps(true);
114 slmSize = std::numeric_limits<uint32_t>::max();
115 getResidencyContainer().clear();
116 getDeallocationContainer().clear();
117 sshAllocations.clear();
118
119 this->handleCmdBufferAllocations(1u);
120 cmdBufferAllocations.erase(cmdBufferAllocations.begin() + 1, cmdBufferAllocations.end());
121
122 commandStream->replaceBuffer(cmdBufferAllocations[0]->getUnderlyingBuffer(),
123 defaultListCmdBufferSize);
124 commandStream->replaceGraphicsAllocation(cmdBufferAllocations[0]);
125 addToResidencyContainer(commandStream->getGraphicsAllocation());
126
127 for (auto &indirectHeap : indirectHeaps) {
128 if (indirectHeap != nullptr) {
129 indirectHeap->replaceBuffer(indirectHeap->getCpuBase(),
130 indirectHeap->getMaxAvailableSpace());
131 addToResidencyContainer(indirectHeap->getGraphicsAllocation());
132 }
133 }
134 if (indirectHeaps[IndirectHeap::Type::SURFACE_STATE] != nullptr) {
135 indirectHeaps[IndirectHeap::Type::SURFACE_STATE]->getSpace(reservedSshSize);
136 }
137
138 iddBlock = nullptr;
139 nextIddInBlock = this->getNumIddPerBlock();
140 lastSentNumGrfRequired = 0;
141 lastPipelineSelectModeRequired = false;
142 lastSentUseGlobalAtomics = false;
143 }
144
getHeapSpaceAllowGrow(HeapType heapType,size_t size)145 void *CommandContainer::getHeapSpaceAllowGrow(HeapType heapType,
146 size_t size) {
147 auto indirectHeap = getIndirectHeap(heapType);
148
149 if (indirectHeap->getAvailableSpace() < size) {
150 size_t newSize = indirectHeap->getUsed() + indirectHeap->getAvailableSpace();
151 newSize *= 2;
152 newSize = std::max(newSize, indirectHeap->getAvailableSpace() + size);
153 newSize = alignUp(newSize, MemoryConstants::pageSize);
154 auto oldAlloc = getIndirectHeapAllocation(heapType);
155 auto newAlloc = getHeapHelper()->getHeapAllocation(heapType, newSize, MemoryConstants::pageSize, device->getRootDeviceIndex());
156 UNRECOVERABLE_IF(!oldAlloc);
157 UNRECOVERABLE_IF(!newAlloc);
158 auto oldBase = indirectHeap->getHeapGpuBase();
159 indirectHeap->replaceGraphicsAllocation(newAlloc);
160 indirectHeap->replaceBuffer(newAlloc->getUnderlyingBuffer(),
161 newAlloc->getUnderlyingBufferSize());
162 auto newBase = indirectHeap->getHeapGpuBase();
163 getResidencyContainer().push_back(newAlloc);
164 getDeallocationContainer().push_back(oldAlloc);
165 setIndirectHeapAllocation(heapType, newAlloc);
166 if (oldBase != newBase) {
167 setHeapDirty(heapType);
168 }
169 }
170 return indirectHeap->getSpace(size);
171 }
172
getHeapWithRequiredSizeAndAlignment(HeapType heapType,size_t sizeRequired,size_t alignment)173 IndirectHeap *CommandContainer::getHeapWithRequiredSizeAndAlignment(HeapType heapType, size_t sizeRequired, size_t alignment) {
174 auto indirectHeap = getIndirectHeap(heapType);
175 auto sizeRequested = sizeRequired;
176
177 auto heapBuffer = indirectHeap->getSpace(0);
178 if (alignment && (heapBuffer != alignUp(heapBuffer, alignment))) {
179 sizeRequested += alignment;
180 }
181
182 if (indirectHeap->getAvailableSpace() < sizeRequested) {
183 size_t newSize = indirectHeap->getUsed() + indirectHeap->getAvailableSpace();
184 newSize = alignUp(newSize, MemoryConstants::pageSize);
185 auto oldAlloc = getIndirectHeapAllocation(heapType);
186 auto newAlloc = getHeapHelper()->getHeapAllocation(heapType, newSize, MemoryConstants::pageSize, device->getRootDeviceIndex());
187 UNRECOVERABLE_IF(!oldAlloc);
188 UNRECOVERABLE_IF(!newAlloc);
189 auto oldBase = indirectHeap->getHeapGpuBase();
190 indirectHeap->replaceGraphicsAllocation(newAlloc);
191 indirectHeap->replaceBuffer(newAlloc->getUnderlyingBuffer(),
192 newAlloc->getUnderlyingBufferSize());
193 auto newBase = indirectHeap->getHeapGpuBase();
194 getResidencyContainer().push_back(newAlloc);
195 getDeallocationContainer().push_back(oldAlloc);
196 setIndirectHeapAllocation(heapType, newAlloc);
197 if (oldBase != newBase) {
198 setHeapDirty(heapType);
199 }
200 if (heapType == HeapType::SURFACE_STATE) {
201 indirectHeap->getSpace(reservedSshSize);
202 sshAllocations.push_back(oldAlloc);
203 }
204 }
205
206 if (alignment) {
207 indirectHeap->align(alignment);
208 }
209
210 return indirectHeap;
211 }
212
handleCmdBufferAllocations(size_t startIndex)213 void CommandContainer::handleCmdBufferAllocations(size_t startIndex) {
214 for (size_t i = startIndex; i < cmdBufferAllocations.size(); i++) {
215 if (this->reusableAllocationList) {
216 this->device->getMemoryManager()->handleFenceCompletion(cmdBufferAllocations[i]);
217 reusableAllocationList->pushFrontOne(*cmdBufferAllocations[i]);
218 } else {
219 this->device->getMemoryManager()->freeGraphicsMemory(cmdBufferAllocations[i]);
220 }
221 }
222 }
223
obtainNextCommandBufferAllocation()224 GraphicsAllocation *CommandContainer::obtainNextCommandBufferAllocation() {
225 size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);
226
227 GraphicsAllocation *cmdBufferAllocation = nullptr;
228 if (this->reusableAllocationList) {
229 cmdBufferAllocation = this->reusableAllocationList->detachAllocation(alignedSize, nullptr, nullptr, GraphicsAllocation::AllocationType::COMMAND_BUFFER).release();
230 }
231 if (!cmdBufferAllocation) {
232 AllocationProperties properties{device->getRootDeviceIndex(),
233 true /* allocateMemory*/,
234 alignedSize,
235 GraphicsAllocation::AllocationType::COMMAND_BUFFER,
236 (device->getNumGenericSubDevices() > 1u) /* multiOsContextCapable */,
237 false,
238 device->getDeviceBitfield()};
239
240 cmdBufferAllocation = device->getMemoryManager()->allocateGraphicsMemoryWithProperties(properties);
241 }
242
243 return cmdBufferAllocation;
244 }
245
allocateNextCommandBuffer()246 void CommandContainer::allocateNextCommandBuffer() {
247 auto cmdBufferAllocation = this->obtainNextCommandBufferAllocation();
248 UNRECOVERABLE_IF(!cmdBufferAllocation);
249
250 cmdBufferAllocations.push_back(cmdBufferAllocation);
251
252 commandStream->replaceBuffer(cmdBufferAllocation->getUnderlyingBuffer(), defaultListCmdBufferSize);
253 commandStream->replaceGraphicsAllocation(cmdBufferAllocation);
254
255 if (!getFlushTaskUsedForImmediate()) {
256 addToResidencyContainer(cmdBufferAllocation);
257 }
258 }
259
prepareBindfulSsh()260 void CommandContainer::prepareBindfulSsh() {
261 if (ApiSpecificConfig::getBindlessConfiguration()) {
262 if (allocationIndirectHeaps[IndirectHeap::SURFACE_STATE] == nullptr) {
263 size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);
264 constexpr size_t heapSize = 65536u;
265 allocationIndirectHeaps[IndirectHeap::SURFACE_STATE] = heapHelper->getHeapAllocation(IndirectHeap::SURFACE_STATE,
266 heapSize,
267 alignedSize,
268 device->getRootDeviceIndex());
269 UNRECOVERABLE_IF(!allocationIndirectHeaps[IndirectHeap::SURFACE_STATE]);
270 residencyContainer.push_back(allocationIndirectHeaps[IndirectHeap::SURFACE_STATE]);
271
272 indirectHeaps[IndirectHeap::SURFACE_STATE] = std::make_unique<IndirectHeap>(allocationIndirectHeaps[IndirectHeap::SURFACE_STATE], false);
273 indirectHeaps[IndirectHeap::SURFACE_STATE]->getSpace(reservedSshSize);
274 }
275 setHeapDirty(IndirectHeap::SURFACE_STATE);
276 }
277 }
278 } // namespace NEO
279