1 /*
2 * Copyright (C) 2019-2021 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8 #include "shared/source/memory_manager/unified_memory_manager.h"
9
10 #include "shared/source/command_stream/command_stream_receiver.h"
11 #include "shared/source/helpers/aligned_memory.h"
12 #include "shared/source/helpers/memory_properties_helpers.h"
13 #include "shared/source/memory_manager/memory_manager.h"
14 #include "shared/source/os_interface/hw_info_config.h"
15
16 namespace NEO {
17
insert(SvmAllocationData allocationsPair)18 void SVMAllocsManager::MapBasedAllocationTracker::insert(SvmAllocationData allocationsPair) {
19 allocations.insert(std::make_pair(reinterpret_cast<void *>(allocationsPair.gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress()), allocationsPair));
20 }
21
remove(SvmAllocationData allocationsPair)22 void SVMAllocsManager::MapBasedAllocationTracker::remove(SvmAllocationData allocationsPair) {
23 SvmAllocationContainer::iterator iter;
24 iter = allocations.find(reinterpret_cast<void *>(allocationsPair.gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress()));
25 allocations.erase(iter);
26 }
27
get(const void * ptr)28 SvmAllocationData *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) {
29 SvmAllocationContainer::iterator Iter, End;
30 SvmAllocationData *svmAllocData;
31 if ((ptr == nullptr) || (allocations.size() == 0)) {
32 return nullptr;
33 }
34 End = allocations.end();
35 Iter = allocations.lower_bound(ptr);
36 if (((Iter != End) && (Iter->first != ptr)) ||
37 (Iter == End)) {
38 if (Iter == allocations.begin()) {
39 Iter = End;
40 } else {
41 Iter--;
42 }
43 }
44 if (Iter != End) {
45 svmAllocData = &Iter->second;
46 char *charPtr = reinterpret_cast<char *>(svmAllocData->gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress());
47 if (ptr < (charPtr + svmAllocData->size)) {
48 return svmAllocData;
49 }
50 }
51 return nullptr;
52 }
53
insert(SvmMapOperation mapOperation)54 void SVMAllocsManager::MapOperationsTracker::insert(SvmMapOperation mapOperation) {
55 operations.insert(std::make_pair(mapOperation.regionSvmPtr, mapOperation));
56 }
57
remove(const void * regionPtr)58 void SVMAllocsManager::MapOperationsTracker::remove(const void *regionPtr) {
59 SvmMapOperationsContainer::iterator iter;
60 iter = operations.find(regionPtr);
61 operations.erase(iter);
62 }
63
get(const void * regionPtr)64 SvmMapOperation *SVMAllocsManager::MapOperationsTracker::get(const void *regionPtr) {
65 SvmMapOperationsContainer::iterator iter;
66 iter = operations.find(regionPtr);
67 if (iter == operations.end()) {
68 return nullptr;
69 }
70 return &iter->second;
71 }
72
addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,ResidencyContainer & residencyContainer,uint32_t requestedTypesMask)73 void SVMAllocsManager::addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,
74 ResidencyContainer &residencyContainer,
75 uint32_t requestedTypesMask) {
76 std::unique_lock<SpinLock> lock(mtx);
77 for (auto &allocation : this->SVMAllocs.allocations) {
78 if (rootDeviceIndex >= allocation.second.gpuAllocations.getGraphicsAllocations().size()) {
79 continue;
80 }
81
82 if (!(allocation.second.memoryType & requestedTypesMask) ||
83 (nullptr == allocation.second.gpuAllocations.getGraphicsAllocation(rootDeviceIndex))) {
84 continue;
85 }
86
87 auto alloc = allocation.second.gpuAllocations.getGraphicsAllocation(rootDeviceIndex);
88 if (residencyContainer.end() == std::find(residencyContainer.begin(), residencyContainer.end(), alloc)) {
89 residencyContainer.push_back(alloc);
90 }
91 }
92 }
93
makeInternalAllocationsResident(CommandStreamReceiver & commandStreamReceiver,uint32_t requestedTypesMask)94 void SVMAllocsManager::makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t requestedTypesMask) {
95 std::unique_lock<SpinLock> lock(mtx);
96 for (auto &allocation : this->SVMAllocs.allocations) {
97 if (allocation.second.memoryType & requestedTypesMask) {
98 auto gpuAllocation = allocation.second.gpuAllocations.getGraphicsAllocation(commandStreamReceiver.getRootDeviceIndex());
99 UNRECOVERABLE_IF(nullptr == gpuAllocation);
100 commandStreamReceiver.makeResident(*gpuAllocation);
101 }
102 }
103 }
104
SVMAllocsManager(MemoryManager * memoryManager,bool multiOsContextSupport)105 SVMAllocsManager::SVMAllocsManager(MemoryManager *memoryManager, bool multiOsContextSupport)
106 : memoryManager(memoryManager), multiOsContextSupport(multiOsContextSupport) {
107 }
108
createSVMAlloc(size_t size,const SvmAllocationProperties svmProperties,const std::set<uint32_t> & rootDeviceIndices,const std::map<uint32_t,DeviceBitfield> & subdeviceBitfields)109 void *SVMAllocsManager::createSVMAlloc(size_t size, const SvmAllocationProperties svmProperties,
110 const std::set<uint32_t> &rootDeviceIndices,
111 const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields) {
112 if (size == 0)
113 return nullptr;
114
115 if (rootDeviceIndices.size() > 1) {
116 return createZeroCopySvmAllocation(size, svmProperties, rootDeviceIndices, subdeviceBitfields);
117 }
118 if (!memoryManager->isLocalMemorySupported(*rootDeviceIndices.begin())) {
119 return createZeroCopySvmAllocation(size, svmProperties, rootDeviceIndices, subdeviceBitfields);
120 } else {
121 UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::NOT_SPECIFIED, rootDeviceIndices, subdeviceBitfields);
122 return createUnifiedAllocationWithDeviceStorage(size, svmProperties, unifiedMemoryProperties);
123 }
124 }
125
createHostUnifiedMemoryAllocation(size_t size,const UnifiedMemoryProperties & memoryProperties)126 void *SVMAllocsManager::createHostUnifiedMemoryAllocation(size_t size,
127 const UnifiedMemoryProperties &memoryProperties) {
128 size_t alignedSize = alignUp<size_t>(size, MemoryConstants::pageSize64k);
129
130 bool compressionEnabled = false;
131 GraphicsAllocation::AllocationType allocationType = getGraphicsAllocationTypeAndCompressionPreference(memoryProperties, compressionEnabled);
132
133 std::vector<uint32_t> rootDeviceIndicesVector(memoryProperties.rootDeviceIndices.begin(), memoryProperties.rootDeviceIndices.end());
134
135 uint32_t rootDeviceIndex = rootDeviceIndicesVector.at(0);
136 auto &deviceBitfield = memoryProperties.subdeviceBitfields.at(rootDeviceIndex);
137
138 AllocationProperties unifiedMemoryProperties{rootDeviceIndex,
139 true,
140 alignedSize,
141 allocationType,
142 false,
143 (deviceBitfield.count() > 1) && multiOsContextSupport,
144 deviceBitfield};
145 unifiedMemoryProperties.flags.preferCompressed = compressionEnabled;
146 unifiedMemoryProperties.flags.shareable = memoryProperties.allocationFlags.flags.shareable;
147 unifiedMemoryProperties.flags.isUSMHostAllocation = true;
148 unifiedMemoryProperties.flags.isUSMDeviceAllocation = false;
149 unifiedMemoryProperties.cacheRegion = MemoryPropertiesHelper::getCacheRegion(memoryProperties.allocationFlags);
150
151 auto maxRootDeviceIndex = *std::max_element(rootDeviceIndicesVector.begin(), rootDeviceIndicesVector.end(), std::less<uint32_t const>());
152 SvmAllocationData allocData(maxRootDeviceIndex);
153
154 void *usmPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector, unifiedMemoryProperties, allocData.gpuAllocations);
155 if (!usmPtr) {
156 return nullptr;
157 }
158
159 allocData.cpuAllocation = nullptr;
160 allocData.size = size;
161 allocData.memoryType = memoryProperties.memoryType;
162 allocData.allocationFlagsProperty = memoryProperties.allocationFlags;
163 allocData.device = nullptr;
164 allocData.setAllocId(this->allocationsCounter++);
165
166 std::unique_lock<SpinLock> lock(mtx);
167 this->SVMAllocs.insert(allocData);
168
169 return usmPtr;
170 }
171
createUnifiedMemoryAllocation(size_t size,const UnifiedMemoryProperties & memoryProperties)172 void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size,
173 const UnifiedMemoryProperties &memoryProperties) {
174 auto rootDeviceIndex = memoryProperties.device
175 ? memoryProperties.device->getRootDeviceIndex()
176 : *memoryProperties.rootDeviceIndices.begin();
177 DeviceBitfield deviceBitfield = memoryProperties.subdeviceBitfields.at(rootDeviceIndex);
178
179 size_t alignedSize = alignUp<size_t>(size, MemoryConstants::pageSize64k);
180
181 bool compressionEnabled = false;
182 GraphicsAllocation::AllocationType allocationType = getGraphicsAllocationTypeAndCompressionPreference(memoryProperties, compressionEnabled);
183
184 bool multiStorageAllocation = (deviceBitfield.count() > 1) && multiOsContextSupport;
185 if ((deviceBitfield.count() > 1) && !multiOsContextSupport) {
186 for (uint32_t i = 0;; i++) {
187 if (deviceBitfield.test(i)) {
188 deviceBitfield.reset();
189 deviceBitfield.set(i);
190 break;
191 }
192 }
193 }
194
195 AllocationProperties unifiedMemoryProperties{rootDeviceIndex,
196 true,
197 alignedSize,
198 allocationType,
199 false,
200 multiStorageAllocation,
201 deviceBitfield};
202 unifiedMemoryProperties.flags.isUSMDeviceAllocation = false;
203 unifiedMemoryProperties.flags.shareable = memoryProperties.allocationFlags.flags.shareable;
204 unifiedMemoryProperties.cacheRegion = MemoryPropertiesHelper::getCacheRegion(memoryProperties.allocationFlags);
205 unifiedMemoryProperties.flags.uncacheable = memoryProperties.allocationFlags.flags.locallyUncachedResource;
206 unifiedMemoryProperties.flags.preferCompressed = compressionEnabled || memoryProperties.allocationFlags.flags.compressedHint;
207
208 if (memoryProperties.memoryType == InternalMemoryType::DEVICE_UNIFIED_MEMORY) {
209 unifiedMemoryProperties.flags.isUSMDeviceAllocation = true;
210 } else if (memoryProperties.memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY) {
211 unifiedMemoryProperties.flags.isUSMHostAllocation = true;
212 }
213
214 GraphicsAllocation *unifiedMemoryAllocation = memoryManager->allocateGraphicsMemoryWithProperties(unifiedMemoryProperties);
215 if (!unifiedMemoryAllocation) {
216 return nullptr;
217 }
218 setUnifiedAllocationProperties(unifiedMemoryAllocation, {});
219
220 SvmAllocationData allocData(rootDeviceIndex);
221 allocData.gpuAllocations.addAllocation(unifiedMemoryAllocation);
222 allocData.cpuAllocation = nullptr;
223 allocData.size = size;
224 allocData.memoryType = memoryProperties.memoryType;
225 allocData.allocationFlagsProperty = memoryProperties.allocationFlags;
226 allocData.device = memoryProperties.device;
227 allocData.setAllocId(this->allocationsCounter++);
228
229 std::unique_lock<SpinLock> lock(mtx);
230 this->SVMAllocs.insert(allocData);
231 return reinterpret_cast<void *>(unifiedMemoryAllocation->getGpuAddress());
232 }
233
createSharedUnifiedMemoryAllocation(size_t size,const UnifiedMemoryProperties & memoryProperties,void * cmdQ)234 void *SVMAllocsManager::createSharedUnifiedMemoryAllocation(size_t size,
235 const UnifiedMemoryProperties &memoryProperties,
236 void *cmdQ) {
237 if (memoryProperties.rootDeviceIndices.size() > 1 && memoryProperties.device == nullptr) {
238 return createHostUnifiedMemoryAllocation(size, memoryProperties);
239 }
240
241 auto supportDualStorageSharedMemory = memoryManager->isLocalMemorySupported(*memoryProperties.rootDeviceIndices.begin());
242
243 if (DebugManager.flags.AllocateSharedAllocationsWithCpuAndGpuStorage.get() != -1) {
244 supportDualStorageSharedMemory = !!DebugManager.flags.AllocateSharedAllocationsWithCpuAndGpuStorage.get();
245 }
246
247 if (supportDualStorageSharedMemory) {
248 bool useKmdMigration = memoryManager->isKmdMigrationAvailable(*memoryProperties.rootDeviceIndices.begin());
249 void *unifiedMemoryPointer = nullptr;
250
251 if (useKmdMigration) {
252 unifiedMemoryPointer = createUnifiedKmdMigratedAllocation(size, {}, memoryProperties);
253 if (!unifiedMemoryPointer) {
254 return nullptr;
255 }
256 } else {
257 unifiedMemoryPointer = createUnifiedAllocationWithDeviceStorage(size, {}, memoryProperties);
258 if (!unifiedMemoryPointer) {
259 return nullptr;
260 }
261
262 UNRECOVERABLE_IF(cmdQ == nullptr);
263 auto pageFaultManager = this->memoryManager->getPageFaultManager();
264 pageFaultManager->insertAllocation(unifiedMemoryPointer, size, this, cmdQ, memoryProperties.allocationFlags);
265 }
266
267 auto unifiedMemoryAllocation = this->getSVMAlloc(unifiedMemoryPointer);
268 unifiedMemoryAllocation->memoryType = memoryProperties.memoryType;
269 unifiedMemoryAllocation->allocationFlagsProperty = memoryProperties.allocationFlags;
270
271 return unifiedMemoryPointer;
272 }
273 return createUnifiedMemoryAllocation(size, memoryProperties);
274 }
275
createUnifiedKmdMigratedAllocation(size_t size,const SvmAllocationProperties & svmProperties,const UnifiedMemoryProperties & unifiedMemoryProperties)276 void *SVMAllocsManager::createUnifiedKmdMigratedAllocation(size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties) {
277
278 auto rootDeviceIndex = unifiedMemoryProperties.device
279 ? unifiedMemoryProperties.device->getRootDeviceIndex()
280 : *unifiedMemoryProperties.rootDeviceIndices.begin();
281 auto &deviceBitfield = unifiedMemoryProperties.subdeviceBitfields.at(rootDeviceIndex);
282 size_t alignedSize = alignUp<size_t>(size, 2 * MemoryConstants::megaByte);
283 AllocationProperties gpuProperties{rootDeviceIndex,
284 true,
285 alignedSize,
286 GraphicsAllocation::AllocationType::UNIFIED_SHARED_MEMORY,
287 false,
288 false,
289 deviceBitfield};
290
291 gpuProperties.alignment = 2 * MemoryConstants::megaByte;
292 auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags);
293 MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false, cacheRegion);
294 GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties);
295 if (!allocationGpu) {
296 return nullptr;
297 }
298 setUnifiedAllocationProperties(allocationGpu, svmProperties);
299
300 SvmAllocationData allocData(rootDeviceIndex);
301 allocData.gpuAllocations.addAllocation(allocationGpu);
302 allocData.cpuAllocation = nullptr;
303 allocData.device = unifiedMemoryProperties.device;
304 allocData.size = size;
305 allocData.setAllocId(this->allocationsCounter++);
306
307 std::unique_lock<SpinLock> lock(mtx);
308 this->SVMAllocs.insert(allocData);
309 return allocationGpu->getUnderlyingBuffer();
310 }
311
setUnifiedAllocationProperties(GraphicsAllocation * allocation,const SvmAllocationProperties & svmProperties)312 void SVMAllocsManager::setUnifiedAllocationProperties(GraphicsAllocation *allocation, const SvmAllocationProperties &svmProperties) {
313 allocation->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly);
314 allocation->setCoherent(svmProperties.coherent);
315 }
316
getSVMAlloc(const void * ptr)317 SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) {
318 std::unique_lock<SpinLock> lock(mtx);
319 return SVMAllocs.get(ptr);
320 }
321
insertSVMAlloc(const SvmAllocationData & svmAllocData)322 void SVMAllocsManager::insertSVMAlloc(const SvmAllocationData &svmAllocData) {
323 std::unique_lock<SpinLock> lock(mtx);
324 SVMAllocs.insert(svmAllocData);
325 }
326
removeSVMAlloc(const SvmAllocationData & svmAllocData)327 void SVMAllocsManager::removeSVMAlloc(const SvmAllocationData &svmAllocData) {
328 std::unique_lock<SpinLock> lock(mtx);
329 SVMAllocs.remove(svmAllocData);
330 }
331
freeSVMAlloc(void * ptr,bool blocking)332 bool SVMAllocsManager::freeSVMAlloc(void *ptr, bool blocking) {
333 SvmAllocationData *svmData = getSVMAlloc(ptr);
334 if (svmData) {
335 if (blocking) {
336 if (svmData->cpuAllocation) {
337 this->memoryManager->waitForEnginesCompletion(*svmData->cpuAllocation);
338 }
339
340 for (auto &gpuAllocation : svmData->gpuAllocations.getGraphicsAllocations()) {
341 if (gpuAllocation) {
342 this->memoryManager->waitForEnginesCompletion(*gpuAllocation);
343 }
344 }
345 }
346
347 auto pageFaultManager = this->memoryManager->getPageFaultManager();
348 if (pageFaultManager) {
349 pageFaultManager->removeAllocation(ptr);
350 }
351 std::unique_lock<SpinLock> lock(mtx);
352 if (svmData->gpuAllocations.getAllocationType() == GraphicsAllocation::AllocationType::SVM_ZERO_COPY) {
353 freeZeroCopySvmAllocation(svmData);
354 } else {
355 freeSvmAllocationWithDeviceStorage(svmData);
356 }
357 return true;
358 }
359 return false;
360 }
361
createZeroCopySvmAllocation(size_t size,const SvmAllocationProperties & svmProperties,const std::set<uint32_t> & rootDeviceIndices,const std::map<uint32_t,DeviceBitfield> & subdeviceBitfields)362 void *SVMAllocsManager::createZeroCopySvmAllocation(size_t size, const SvmAllocationProperties &svmProperties,
363 const std::set<uint32_t> &rootDeviceIndices,
364 const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields) {
365
366 auto rootDeviceIndex = *rootDeviceIndices.begin();
367 auto &deviceBitfield = subdeviceBitfields.at(rootDeviceIndex);
368 AllocationProperties properties{rootDeviceIndex,
369 true, // allocateMemory
370 size,
371 GraphicsAllocation::AllocationType::SVM_ZERO_COPY,
372 false, // isMultiStorageAllocation
373 deviceBitfield};
374 MemoryPropertiesHelper::fillCachePolicyInProperties(properties, false, svmProperties.readOnly, false, properties.cacheRegion);
375
376 std::vector<uint32_t> rootDeviceIndicesVector(rootDeviceIndices.begin(), rootDeviceIndices.end());
377
378 auto maxRootDeviceIndex = *std::max_element(rootDeviceIndices.begin(), rootDeviceIndices.end(), std::less<uint32_t const>());
379 SvmAllocationData allocData(maxRootDeviceIndex);
380
381 void *usmPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector, properties, allocData.gpuAllocations);
382 if (!usmPtr) {
383 return nullptr;
384 }
385 for (const auto &rootDeviceIndex : rootDeviceIndices) {
386 auto allocation = allocData.gpuAllocations.getGraphicsAllocation(rootDeviceIndex);
387 allocation->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly);
388 allocation->setCoherent(svmProperties.coherent);
389 }
390 allocData.size = size;
391
392 std::unique_lock<SpinLock> lock(mtx);
393 this->SVMAllocs.insert(allocData);
394 return usmPtr;
395 }
396
createUnifiedAllocationWithDeviceStorage(size_t size,const SvmAllocationProperties & svmProperties,const UnifiedMemoryProperties & unifiedMemoryProperties)397 void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties) {
398 auto rootDeviceIndex = unifiedMemoryProperties.device
399 ? unifiedMemoryProperties.device->getRootDeviceIndex()
400 : *unifiedMemoryProperties.rootDeviceIndices.begin();
401 size_t alignedSizeCpu = alignUp<size_t>(size, MemoryConstants::pageSize2Mb);
402 size_t alignedSizeGpu = alignUp<size_t>(size, MemoryConstants::pageSize64k);
403 DeviceBitfield subDevices = unifiedMemoryProperties.subdeviceBitfields.at(rootDeviceIndex);
404 AllocationProperties cpuProperties{rootDeviceIndex,
405 true, // allocateMemory
406 alignedSizeCpu, GraphicsAllocation::AllocationType::SVM_CPU,
407 false, // isMultiStorageAllocation
408 subDevices};
409 cpuProperties.alignment = MemoryConstants::pageSize2Mb;
410 auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags);
411 MemoryPropertiesHelper::fillCachePolicyInProperties(cpuProperties, false, svmProperties.readOnly, false, cacheRegion);
412 GraphicsAllocation *allocationCpu = memoryManager->allocateGraphicsMemoryWithProperties(cpuProperties);
413 if (!allocationCpu) {
414 return nullptr;
415 }
416 setUnifiedAllocationProperties(allocationCpu, svmProperties);
417 void *svmPtr = allocationCpu->getUnderlyingBuffer();
418
419 bool multiStorageAllocation = (subDevices.count() > 1) && multiOsContextSupport;
420 if ((subDevices.count() > 1) && !multiOsContextSupport) {
421 for (uint32_t i = 0;; i++) {
422 if (subDevices.test(i)) {
423 subDevices.reset();
424 subDevices.set(i);
425 break;
426 }
427 }
428 }
429
430 AllocationProperties gpuProperties{rootDeviceIndex,
431 false,
432 alignedSizeGpu,
433 GraphicsAllocation::AllocationType::SVM_GPU,
434 false,
435 multiStorageAllocation,
436 subDevices};
437
438 gpuProperties.alignment = MemoryConstants::pageSize64k;
439 MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false, cacheRegion);
440 GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties, svmPtr);
441 if (!allocationGpu) {
442 memoryManager->freeGraphicsMemory(allocationCpu);
443 return nullptr;
444 }
445 setUnifiedAllocationProperties(allocationGpu, svmProperties);
446
447 SvmAllocationData allocData(rootDeviceIndex);
448 allocData.gpuAllocations.addAllocation(allocationGpu);
449 allocData.cpuAllocation = allocationCpu;
450 allocData.device = unifiedMemoryProperties.device;
451 allocData.size = size;
452 allocData.setAllocId(this->allocationsCounter++);
453
454 std::unique_lock<SpinLock> lock(mtx);
455 this->SVMAllocs.insert(allocData);
456 return svmPtr;
457 }
458
freeZeroCopySvmAllocation(SvmAllocationData * svmData)459 void SVMAllocsManager::freeZeroCopySvmAllocation(SvmAllocationData *svmData) {
460 auto gpuAllocations = svmData->gpuAllocations;
461 SVMAllocs.remove(*svmData);
462 for (const auto &graphicsAllocation : gpuAllocations.getGraphicsAllocations()) {
463 memoryManager->freeGraphicsMemory(graphicsAllocation);
464 }
465 }
466
freeSvmAllocationWithDeviceStorage(SvmAllocationData * svmData)467 void SVMAllocsManager::freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData) {
468 auto graphicsAllocations = svmData->gpuAllocations.getGraphicsAllocations();
469 GraphicsAllocation *cpuAllocation = svmData->cpuAllocation;
470 SVMAllocs.remove(*svmData);
471
472 for (auto gpuAllocation : graphicsAllocations) {
473 memoryManager->freeGraphicsMemory(gpuAllocation);
474 }
475 memoryManager->freeGraphicsMemory(cpuAllocation);
476 }
477
hasHostAllocations()478 bool SVMAllocsManager::hasHostAllocations() {
479 std::unique_lock<SpinLock> lock(mtx);
480 for (auto &allocation : this->SVMAllocs.allocations) {
481 if (allocation.second.memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY) {
482 return true;
483 }
484 }
485 return false;
486 }
487
getSvmMapOperation(const void * ptr)488 SvmMapOperation *SVMAllocsManager::getSvmMapOperation(const void *ptr) {
489 std::unique_lock<SpinLock> lock(mtx);
490 return svmMapOperations.get(ptr);
491 }
492
insertSvmMapOperation(void * regionSvmPtr,size_t regionSize,void * baseSvmPtr,size_t offset,bool readOnlyMap)493 void SVMAllocsManager::insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap) {
494 SvmMapOperation svmMapOperation;
495 svmMapOperation.regionSvmPtr = regionSvmPtr;
496 svmMapOperation.baseSvmPtr = baseSvmPtr;
497 svmMapOperation.offset = offset;
498 svmMapOperation.regionSize = regionSize;
499 svmMapOperation.readOnlyMap = readOnlyMap;
500 std::unique_lock<SpinLock> lock(mtx);
501 svmMapOperations.insert(svmMapOperation);
502 }
503
removeSvmMapOperation(const void * regionSvmPtr)504 void SVMAllocsManager::removeSvmMapOperation(const void *regionSvmPtr) {
505 std::unique_lock<SpinLock> lock(mtx);
506 svmMapOperations.remove(regionSvmPtr);
507 }
508
getGraphicsAllocationTypeAndCompressionPreference(const UnifiedMemoryProperties & unifiedMemoryProperties,bool & compressionEnabled) const509 GraphicsAllocation::AllocationType SVMAllocsManager::getGraphicsAllocationTypeAndCompressionPreference(const UnifiedMemoryProperties &unifiedMemoryProperties, bool &compressionEnabled) const {
510 compressionEnabled = false;
511
512 GraphicsAllocation::AllocationType allocationType = GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY;
513 if (unifiedMemoryProperties.memoryType == InternalMemoryType::DEVICE_UNIFIED_MEMORY) {
514 if (unifiedMemoryProperties.allocationFlags.allocFlags.allocWriteCombined) {
515 allocationType = GraphicsAllocation::AllocationType::WRITE_COMBINED;
516 } else {
517 UNRECOVERABLE_IF(nullptr == unifiedMemoryProperties.device);
518 const auto &hwInfoConfig = *HwInfoConfig::get(unifiedMemoryProperties.device->getHardwareInfo().platform.eProductFamily);
519 if (hwInfoConfig.allowStatelessCompression(unifiedMemoryProperties.device->getHardwareInfo())) {
520 compressionEnabled = true;
521 }
522 allocationType = GraphicsAllocation::AllocationType::BUFFER;
523 }
524 }
525 return allocationType;
526 }
527
528 } // namespace NEO
529