1 /*
2  * Copyright (C) 2019-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #include "shared/source/command_container/cmdcontainer.h"
9 
10 #include "shared/source/command_container/command_encoder.h"
11 #include "shared/source/command_stream/command_stream_receiver.h"
12 #include "shared/source/command_stream/linear_stream.h"
13 #include "shared/source/device/device.h"
14 #include "shared/source/helpers/api_specific_config.h"
15 #include "shared/source/helpers/debug_helpers.h"
16 #include "shared/source/helpers/heap_helper.h"
17 #include "shared/source/helpers/hw_helper.h"
18 #include "shared/source/indirect_heap/indirect_heap.h"
19 #include "shared/source/memory_manager/allocations_list.h"
20 #include "shared/source/memory_manager/memory_manager.h"
21 
22 namespace NEO {
23 
~CommandContainer()24 CommandContainer::~CommandContainer() {
25     if (!device) {
26         DEBUG_BREAK_IF(device);
27         return;
28     }
29 
30     this->handleCmdBufferAllocations(0u);
31 
32     for (auto allocationIndirectHeap : allocationIndirectHeaps) {
33         if (heapHelper) {
34             heapHelper->storeHeapAllocation(allocationIndirectHeap);
35         }
36     }
37     for (auto deallocation : deallocationContainer) {
38         if (((deallocation->getAllocationType() == GraphicsAllocation::AllocationType::INTERNAL_HEAP) || (deallocation->getAllocationType() == GraphicsAllocation::AllocationType::LINEAR_STREAM))) {
39             getHeapHelper()->storeHeapAllocation(deallocation);
40         }
41     }
42 }
43 
initialize(Device * device,AllocationsList * reusableAllocationList)44 ErrorCode CommandContainer::initialize(Device *device, AllocationsList *reusableAllocationList) {
45     this->device = device;
46     this->reusableAllocationList = reusableAllocationList;
47 
48     size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);
49 
50     auto cmdBufferAllocation = this->obtainNextCommandBufferAllocation();
51 
52     if (!cmdBufferAllocation) {
53         return ErrorCode::OUT_OF_DEVICE_MEMORY;
54     }
55 
56     cmdBufferAllocations.push_back(cmdBufferAllocation);
57 
58     commandStream = std::unique_ptr<LinearStream>(new LinearStream(cmdBufferAllocation->getUnderlyingBuffer(),
59                                                                    defaultListCmdBufferSize));
60     commandStream->replaceGraphicsAllocation(cmdBufferAllocation);
61 
62     if (!getFlushTaskUsedForImmediate()) {
63         addToResidencyContainer(cmdBufferAllocation);
64     }
65 
66     constexpr size_t heapSize = 65536u;
67     heapHelper = std::unique_ptr<HeapHelper>(new HeapHelper(device->getMemoryManager(), device->getDefaultEngine().commandStreamReceiver->getInternalAllocationStorage(), device->getNumGenericSubDevices() > 1u));
68 
69     for (uint32_t i = 0; i < IndirectHeap::Type::NUM_TYPES; i++) {
70         if (NEO::ApiSpecificConfig::getBindlessConfiguration() && i != IndirectHeap::INDIRECT_OBJECT) {
71             continue;
72         }
73         allocationIndirectHeaps[i] = heapHelper->getHeapAllocation(i,
74                                                                    heapSize,
75                                                                    alignedSize,
76                                                                    device->getRootDeviceIndex());
77         if (!allocationIndirectHeaps[i]) {
78             return ErrorCode::OUT_OF_DEVICE_MEMORY;
79         }
80         residencyContainer.push_back(allocationIndirectHeaps[i]);
81 
82         bool requireInternalHeap = (IndirectHeap::INDIRECT_OBJECT == i);
83         indirectHeaps[i] = std::make_unique<IndirectHeap>(allocationIndirectHeaps[i], requireInternalHeap);
84         if (i == IndirectHeap::Type::SURFACE_STATE) {
85             indirectHeaps[i]->getSpace(reservedSshSize);
86         }
87     }
88 
89     indirectObjectHeapBaseAddress = device->getMemoryManager()->getInternalHeapBaseAddress(device->getRootDeviceIndex(), allocationIndirectHeaps[IndirectHeap::Type::INDIRECT_OBJECT]->isAllocatedInLocalMemoryPool());
90 
91     instructionHeapBaseAddress = device->getMemoryManager()->getInternalHeapBaseAddress(device->getRootDeviceIndex(), device->getMemoryManager()->isLocalMemoryUsedForIsa(device->getRootDeviceIndex()));
92 
93     iddBlock = nullptr;
94     nextIddInBlock = this->getNumIddPerBlock();
95 
96     return ErrorCode::SUCCESS;
97 }
98 
addToResidencyContainer(GraphicsAllocation * alloc)99 void CommandContainer::addToResidencyContainer(GraphicsAllocation *alloc) {
100     if (alloc == nullptr) {
101         return;
102     }
103 
104     this->residencyContainer.push_back(alloc);
105 }
106 
removeDuplicatesFromResidencyContainer()107 void CommandContainer::removeDuplicatesFromResidencyContainer() {
108     std::sort(this->residencyContainer.begin(), this->residencyContainer.end());
109     this->residencyContainer.erase(std::unique(this->residencyContainer.begin(), this->residencyContainer.end()), this->residencyContainer.end());
110 }
111 
reset()112 void CommandContainer::reset() {
113     setDirtyStateForAllHeaps(true);
114     slmSize = std::numeric_limits<uint32_t>::max();
115     getResidencyContainer().clear();
116     getDeallocationContainer().clear();
117     sshAllocations.clear();
118 
119     this->handleCmdBufferAllocations(1u);
120     cmdBufferAllocations.erase(cmdBufferAllocations.begin() + 1, cmdBufferAllocations.end());
121 
122     commandStream->replaceBuffer(cmdBufferAllocations[0]->getUnderlyingBuffer(),
123                                  defaultListCmdBufferSize);
124     commandStream->replaceGraphicsAllocation(cmdBufferAllocations[0]);
125     addToResidencyContainer(commandStream->getGraphicsAllocation());
126 
127     for (auto &indirectHeap : indirectHeaps) {
128         if (indirectHeap != nullptr) {
129             indirectHeap->replaceBuffer(indirectHeap->getCpuBase(),
130                                         indirectHeap->getMaxAvailableSpace());
131             addToResidencyContainer(indirectHeap->getGraphicsAllocation());
132         }
133     }
134     if (indirectHeaps[IndirectHeap::Type::SURFACE_STATE] != nullptr) {
135         indirectHeaps[IndirectHeap::Type::SURFACE_STATE]->getSpace(reservedSshSize);
136     }
137 
138     iddBlock = nullptr;
139     nextIddInBlock = this->getNumIddPerBlock();
140     lastSentNumGrfRequired = 0;
141     lastPipelineSelectModeRequired = false;
142     lastSentUseGlobalAtomics = false;
143 }
144 
getHeapSpaceAllowGrow(HeapType heapType,size_t size)145 void *CommandContainer::getHeapSpaceAllowGrow(HeapType heapType,
146                                               size_t size) {
147     auto indirectHeap = getIndirectHeap(heapType);
148 
149     if (indirectHeap->getAvailableSpace() < size) {
150         size_t newSize = indirectHeap->getUsed() + indirectHeap->getAvailableSpace();
151         newSize *= 2;
152         newSize = std::max(newSize, indirectHeap->getAvailableSpace() + size);
153         newSize = alignUp(newSize, MemoryConstants::pageSize);
154         auto oldAlloc = getIndirectHeapAllocation(heapType);
155         auto newAlloc = getHeapHelper()->getHeapAllocation(heapType, newSize, MemoryConstants::pageSize, device->getRootDeviceIndex());
156         UNRECOVERABLE_IF(!oldAlloc);
157         UNRECOVERABLE_IF(!newAlloc);
158         auto oldBase = indirectHeap->getHeapGpuBase();
159         indirectHeap->replaceGraphicsAllocation(newAlloc);
160         indirectHeap->replaceBuffer(newAlloc->getUnderlyingBuffer(),
161                                     newAlloc->getUnderlyingBufferSize());
162         auto newBase = indirectHeap->getHeapGpuBase();
163         getResidencyContainer().push_back(newAlloc);
164         getDeallocationContainer().push_back(oldAlloc);
165         setIndirectHeapAllocation(heapType, newAlloc);
166         if (oldBase != newBase) {
167             setHeapDirty(heapType);
168         }
169     }
170     return indirectHeap->getSpace(size);
171 }
172 
getHeapWithRequiredSizeAndAlignment(HeapType heapType,size_t sizeRequired,size_t alignment)173 IndirectHeap *CommandContainer::getHeapWithRequiredSizeAndAlignment(HeapType heapType, size_t sizeRequired, size_t alignment) {
174     auto indirectHeap = getIndirectHeap(heapType);
175     auto sizeRequested = sizeRequired;
176 
177     auto heapBuffer = indirectHeap->getSpace(0);
178     if (alignment && (heapBuffer != alignUp(heapBuffer, alignment))) {
179         sizeRequested += alignment;
180     }
181 
182     if (indirectHeap->getAvailableSpace() < sizeRequested) {
183         size_t newSize = indirectHeap->getUsed() + indirectHeap->getAvailableSpace();
184         newSize = alignUp(newSize, MemoryConstants::pageSize);
185         auto oldAlloc = getIndirectHeapAllocation(heapType);
186         auto newAlloc = getHeapHelper()->getHeapAllocation(heapType, newSize, MemoryConstants::pageSize, device->getRootDeviceIndex());
187         UNRECOVERABLE_IF(!oldAlloc);
188         UNRECOVERABLE_IF(!newAlloc);
189         auto oldBase = indirectHeap->getHeapGpuBase();
190         indirectHeap->replaceGraphicsAllocation(newAlloc);
191         indirectHeap->replaceBuffer(newAlloc->getUnderlyingBuffer(),
192                                     newAlloc->getUnderlyingBufferSize());
193         auto newBase = indirectHeap->getHeapGpuBase();
194         getResidencyContainer().push_back(newAlloc);
195         getDeallocationContainer().push_back(oldAlloc);
196         setIndirectHeapAllocation(heapType, newAlloc);
197         if (oldBase != newBase) {
198             setHeapDirty(heapType);
199         }
200         if (heapType == HeapType::SURFACE_STATE) {
201             indirectHeap->getSpace(reservedSshSize);
202             sshAllocations.push_back(oldAlloc);
203         }
204     }
205 
206     if (alignment) {
207         indirectHeap->align(alignment);
208     }
209 
210     return indirectHeap;
211 }
212 
handleCmdBufferAllocations(size_t startIndex)213 void CommandContainer::handleCmdBufferAllocations(size_t startIndex) {
214     for (size_t i = startIndex; i < cmdBufferAllocations.size(); i++) {
215         if (this->reusableAllocationList) {
216             this->device->getMemoryManager()->handleFenceCompletion(cmdBufferAllocations[i]);
217             reusableAllocationList->pushFrontOne(*cmdBufferAllocations[i]);
218         } else {
219             this->device->getMemoryManager()->freeGraphicsMemory(cmdBufferAllocations[i]);
220         }
221     }
222 }
223 
obtainNextCommandBufferAllocation()224 GraphicsAllocation *CommandContainer::obtainNextCommandBufferAllocation() {
225     size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);
226 
227     GraphicsAllocation *cmdBufferAllocation = nullptr;
228     if (this->reusableAllocationList) {
229         cmdBufferAllocation = this->reusableAllocationList->detachAllocation(alignedSize, nullptr, nullptr, GraphicsAllocation::AllocationType::COMMAND_BUFFER).release();
230     }
231     if (!cmdBufferAllocation) {
232         AllocationProperties properties{device->getRootDeviceIndex(),
233                                         true /* allocateMemory*/,
234                                         alignedSize,
235                                         GraphicsAllocation::AllocationType::COMMAND_BUFFER,
236                                         (device->getNumGenericSubDevices() > 1u) /* multiOsContextCapable */,
237                                         false,
238                                         device->getDeviceBitfield()};
239 
240         cmdBufferAllocation = device->getMemoryManager()->allocateGraphicsMemoryWithProperties(properties);
241     }
242 
243     return cmdBufferAllocation;
244 }
245 
allocateNextCommandBuffer()246 void CommandContainer::allocateNextCommandBuffer() {
247     auto cmdBufferAllocation = this->obtainNextCommandBufferAllocation();
248     UNRECOVERABLE_IF(!cmdBufferAllocation);
249 
250     cmdBufferAllocations.push_back(cmdBufferAllocation);
251 
252     commandStream->replaceBuffer(cmdBufferAllocation->getUnderlyingBuffer(), defaultListCmdBufferSize);
253     commandStream->replaceGraphicsAllocation(cmdBufferAllocation);
254 
255     if (!getFlushTaskUsedForImmediate()) {
256         addToResidencyContainer(cmdBufferAllocation);
257     }
258 }
259 
prepareBindfulSsh()260 void CommandContainer::prepareBindfulSsh() {
261     if (ApiSpecificConfig::getBindlessConfiguration()) {
262         if (allocationIndirectHeaps[IndirectHeap::SURFACE_STATE] == nullptr) {
263             size_t alignedSize = alignUp<size_t>(totalCmdBufferSize, MemoryConstants::pageSize64k);
264             constexpr size_t heapSize = 65536u;
265             allocationIndirectHeaps[IndirectHeap::SURFACE_STATE] = heapHelper->getHeapAllocation(IndirectHeap::SURFACE_STATE,
266                                                                                                  heapSize,
267                                                                                                  alignedSize,
268                                                                                                  device->getRootDeviceIndex());
269             UNRECOVERABLE_IF(!allocationIndirectHeaps[IndirectHeap::SURFACE_STATE]);
270             residencyContainer.push_back(allocationIndirectHeaps[IndirectHeap::SURFACE_STATE]);
271 
272             indirectHeaps[IndirectHeap::SURFACE_STATE] = std::make_unique<IndirectHeap>(allocationIndirectHeaps[IndirectHeap::SURFACE_STATE], false);
273             indirectHeaps[IndirectHeap::SURFACE_STATE]->getSpace(reservedSshSize);
274         }
275         setHeapDirty(IndirectHeap::SURFACE_STATE);
276     }
277 }
278 } // namespace NEO
279