1 /*
2  * Copyright (C) 2019-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #include "shared/source/memory_manager/unified_memory_manager.h"
9 
10 #include "shared/source/command_stream/command_stream_receiver.h"
11 #include "shared/source/helpers/aligned_memory.h"
12 #include "shared/source/helpers/memory_properties_helpers.h"
13 #include "shared/source/memory_manager/memory_manager.h"
14 #include "shared/source/os_interface/hw_info_config.h"
15 
16 namespace NEO {
17 
insert(SvmAllocationData allocationsPair)18 void SVMAllocsManager::MapBasedAllocationTracker::insert(SvmAllocationData allocationsPair) {
19     allocations.insert(std::make_pair(reinterpret_cast<void *>(allocationsPair.gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress()), allocationsPair));
20 }
21 
remove(SvmAllocationData allocationsPair)22 void SVMAllocsManager::MapBasedAllocationTracker::remove(SvmAllocationData allocationsPair) {
23     SvmAllocationContainer::iterator iter;
24     iter = allocations.find(reinterpret_cast<void *>(allocationsPair.gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress()));
25     allocations.erase(iter);
26 }
27 
get(const void * ptr)28 SvmAllocationData *SVMAllocsManager::MapBasedAllocationTracker::get(const void *ptr) {
29     SvmAllocationContainer::iterator Iter, End;
30     SvmAllocationData *svmAllocData;
31     if ((ptr == nullptr) || (allocations.size() == 0)) {
32         return nullptr;
33     }
34     End = allocations.end();
35     Iter = allocations.lower_bound(ptr);
36     if (((Iter != End) && (Iter->first != ptr)) ||
37         (Iter == End)) {
38         if (Iter == allocations.begin()) {
39             Iter = End;
40         } else {
41             Iter--;
42         }
43     }
44     if (Iter != End) {
45         svmAllocData = &Iter->second;
46         char *charPtr = reinterpret_cast<char *>(svmAllocData->gpuAllocations.getDefaultGraphicsAllocation()->getGpuAddress());
47         if (ptr < (charPtr + svmAllocData->size)) {
48             return svmAllocData;
49         }
50     }
51     return nullptr;
52 }
53 
insert(SvmMapOperation mapOperation)54 void SVMAllocsManager::MapOperationsTracker::insert(SvmMapOperation mapOperation) {
55     operations.insert(std::make_pair(mapOperation.regionSvmPtr, mapOperation));
56 }
57 
remove(const void * regionPtr)58 void SVMAllocsManager::MapOperationsTracker::remove(const void *regionPtr) {
59     SvmMapOperationsContainer::iterator iter;
60     iter = operations.find(regionPtr);
61     operations.erase(iter);
62 }
63 
get(const void * regionPtr)64 SvmMapOperation *SVMAllocsManager::MapOperationsTracker::get(const void *regionPtr) {
65     SvmMapOperationsContainer::iterator iter;
66     iter = operations.find(regionPtr);
67     if (iter == operations.end()) {
68         return nullptr;
69     }
70     return &iter->second;
71 }
72 
addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,ResidencyContainer & residencyContainer,uint32_t requestedTypesMask)73 void SVMAllocsManager::addInternalAllocationsToResidencyContainer(uint32_t rootDeviceIndex,
74                                                                   ResidencyContainer &residencyContainer,
75                                                                   uint32_t requestedTypesMask) {
76     std::unique_lock<SpinLock> lock(mtx);
77     for (auto &allocation : this->SVMAllocs.allocations) {
78         if (rootDeviceIndex >= allocation.second.gpuAllocations.getGraphicsAllocations().size()) {
79             continue;
80         }
81 
82         if (!(allocation.second.memoryType & requestedTypesMask) ||
83             (nullptr == allocation.second.gpuAllocations.getGraphicsAllocation(rootDeviceIndex))) {
84             continue;
85         }
86 
87         auto alloc = allocation.second.gpuAllocations.getGraphicsAllocation(rootDeviceIndex);
88         if (residencyContainer.end() == std::find(residencyContainer.begin(), residencyContainer.end(), alloc)) {
89             residencyContainer.push_back(alloc);
90         }
91     }
92 }
93 
makeInternalAllocationsResident(CommandStreamReceiver & commandStreamReceiver,uint32_t requestedTypesMask)94 void SVMAllocsManager::makeInternalAllocationsResident(CommandStreamReceiver &commandStreamReceiver, uint32_t requestedTypesMask) {
95     std::unique_lock<SpinLock> lock(mtx);
96     for (auto &allocation : this->SVMAllocs.allocations) {
97         if (allocation.second.memoryType & requestedTypesMask) {
98             auto gpuAllocation = allocation.second.gpuAllocations.getGraphicsAllocation(commandStreamReceiver.getRootDeviceIndex());
99             UNRECOVERABLE_IF(nullptr == gpuAllocation);
100             commandStreamReceiver.makeResident(*gpuAllocation);
101         }
102     }
103 }
104 
SVMAllocsManager(MemoryManager * memoryManager,bool multiOsContextSupport)105 SVMAllocsManager::SVMAllocsManager(MemoryManager *memoryManager, bool multiOsContextSupport)
106     : memoryManager(memoryManager), multiOsContextSupport(multiOsContextSupport) {
107 }
108 
createSVMAlloc(size_t size,const SvmAllocationProperties svmProperties,const std::set<uint32_t> & rootDeviceIndices,const std::map<uint32_t,DeviceBitfield> & subdeviceBitfields)109 void *SVMAllocsManager::createSVMAlloc(size_t size, const SvmAllocationProperties svmProperties,
110                                        const std::set<uint32_t> &rootDeviceIndices,
111                                        const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields) {
112     if (size == 0)
113         return nullptr;
114 
115     if (rootDeviceIndices.size() > 1) {
116         return createZeroCopySvmAllocation(size, svmProperties, rootDeviceIndices, subdeviceBitfields);
117     }
118     if (!memoryManager->isLocalMemorySupported(*rootDeviceIndices.begin())) {
119         return createZeroCopySvmAllocation(size, svmProperties, rootDeviceIndices, subdeviceBitfields);
120     } else {
121         UnifiedMemoryProperties unifiedMemoryProperties(InternalMemoryType::NOT_SPECIFIED, rootDeviceIndices, subdeviceBitfields);
122         return createUnifiedAllocationWithDeviceStorage(size, svmProperties, unifiedMemoryProperties);
123     }
124 }
125 
createHostUnifiedMemoryAllocation(size_t size,const UnifiedMemoryProperties & memoryProperties)126 void *SVMAllocsManager::createHostUnifiedMemoryAllocation(size_t size,
127                                                           const UnifiedMemoryProperties &memoryProperties) {
128     size_t alignedSize = alignUp<size_t>(size, MemoryConstants::pageSize64k);
129 
130     bool compressionEnabled = false;
131     GraphicsAllocation::AllocationType allocationType = getGraphicsAllocationTypeAndCompressionPreference(memoryProperties, compressionEnabled);
132 
133     std::vector<uint32_t> rootDeviceIndicesVector(memoryProperties.rootDeviceIndices.begin(), memoryProperties.rootDeviceIndices.end());
134 
135     uint32_t rootDeviceIndex = rootDeviceIndicesVector.at(0);
136     auto &deviceBitfield = memoryProperties.subdeviceBitfields.at(rootDeviceIndex);
137 
138     AllocationProperties unifiedMemoryProperties{rootDeviceIndex,
139                                                  true,
140                                                  alignedSize,
141                                                  allocationType,
142                                                  false,
143                                                  (deviceBitfield.count() > 1) && multiOsContextSupport,
144                                                  deviceBitfield};
145     unifiedMemoryProperties.flags.preferCompressed = compressionEnabled;
146     unifiedMemoryProperties.flags.shareable = memoryProperties.allocationFlags.flags.shareable;
147     unifiedMemoryProperties.flags.isUSMHostAllocation = true;
148     unifiedMemoryProperties.flags.isUSMDeviceAllocation = false;
149     unifiedMemoryProperties.cacheRegion = MemoryPropertiesHelper::getCacheRegion(memoryProperties.allocationFlags);
150 
151     auto maxRootDeviceIndex = *std::max_element(rootDeviceIndicesVector.begin(), rootDeviceIndicesVector.end(), std::less<uint32_t const>());
152     SvmAllocationData allocData(maxRootDeviceIndex);
153 
154     void *usmPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector, unifiedMemoryProperties, allocData.gpuAllocations);
155     if (!usmPtr) {
156         return nullptr;
157     }
158 
159     allocData.cpuAllocation = nullptr;
160     allocData.size = size;
161     allocData.memoryType = memoryProperties.memoryType;
162     allocData.allocationFlagsProperty = memoryProperties.allocationFlags;
163     allocData.device = nullptr;
164     allocData.setAllocId(this->allocationsCounter++);
165 
166     std::unique_lock<SpinLock> lock(mtx);
167     this->SVMAllocs.insert(allocData);
168 
169     return usmPtr;
170 }
171 
createUnifiedMemoryAllocation(size_t size,const UnifiedMemoryProperties & memoryProperties)172 void *SVMAllocsManager::createUnifiedMemoryAllocation(size_t size,
173                                                       const UnifiedMemoryProperties &memoryProperties) {
174     auto rootDeviceIndex = memoryProperties.device
175                                ? memoryProperties.device->getRootDeviceIndex()
176                                : *memoryProperties.rootDeviceIndices.begin();
177     DeviceBitfield deviceBitfield = memoryProperties.subdeviceBitfields.at(rootDeviceIndex);
178 
179     size_t alignedSize = alignUp<size_t>(size, MemoryConstants::pageSize64k);
180 
181     bool compressionEnabled = false;
182     GraphicsAllocation::AllocationType allocationType = getGraphicsAllocationTypeAndCompressionPreference(memoryProperties, compressionEnabled);
183 
184     bool multiStorageAllocation = (deviceBitfield.count() > 1) && multiOsContextSupport;
185     if ((deviceBitfield.count() > 1) && !multiOsContextSupport) {
186         for (uint32_t i = 0;; i++) {
187             if (deviceBitfield.test(i)) {
188                 deviceBitfield.reset();
189                 deviceBitfield.set(i);
190                 break;
191             }
192         }
193     }
194 
195     AllocationProperties unifiedMemoryProperties{rootDeviceIndex,
196                                                  true,
197                                                  alignedSize,
198                                                  allocationType,
199                                                  false,
200                                                  multiStorageAllocation,
201                                                  deviceBitfield};
202     unifiedMemoryProperties.flags.isUSMDeviceAllocation = false;
203     unifiedMemoryProperties.flags.shareable = memoryProperties.allocationFlags.flags.shareable;
204     unifiedMemoryProperties.cacheRegion = MemoryPropertiesHelper::getCacheRegion(memoryProperties.allocationFlags);
205     unifiedMemoryProperties.flags.uncacheable = memoryProperties.allocationFlags.flags.locallyUncachedResource;
206     unifiedMemoryProperties.flags.preferCompressed = compressionEnabled || memoryProperties.allocationFlags.flags.compressedHint;
207 
208     if (memoryProperties.memoryType == InternalMemoryType::DEVICE_UNIFIED_MEMORY) {
209         unifiedMemoryProperties.flags.isUSMDeviceAllocation = true;
210     } else if (memoryProperties.memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY) {
211         unifiedMemoryProperties.flags.isUSMHostAllocation = true;
212     }
213 
214     GraphicsAllocation *unifiedMemoryAllocation = memoryManager->allocateGraphicsMemoryWithProperties(unifiedMemoryProperties);
215     if (!unifiedMemoryAllocation) {
216         return nullptr;
217     }
218     setUnifiedAllocationProperties(unifiedMemoryAllocation, {});
219 
220     SvmAllocationData allocData(rootDeviceIndex);
221     allocData.gpuAllocations.addAllocation(unifiedMemoryAllocation);
222     allocData.cpuAllocation = nullptr;
223     allocData.size = size;
224     allocData.memoryType = memoryProperties.memoryType;
225     allocData.allocationFlagsProperty = memoryProperties.allocationFlags;
226     allocData.device = memoryProperties.device;
227     allocData.setAllocId(this->allocationsCounter++);
228 
229     std::unique_lock<SpinLock> lock(mtx);
230     this->SVMAllocs.insert(allocData);
231     return reinterpret_cast<void *>(unifiedMemoryAllocation->getGpuAddress());
232 }
233 
createSharedUnifiedMemoryAllocation(size_t size,const UnifiedMemoryProperties & memoryProperties,void * cmdQ)234 void *SVMAllocsManager::createSharedUnifiedMemoryAllocation(size_t size,
235                                                             const UnifiedMemoryProperties &memoryProperties,
236                                                             void *cmdQ) {
237     if (memoryProperties.rootDeviceIndices.size() > 1 && memoryProperties.device == nullptr) {
238         return createHostUnifiedMemoryAllocation(size, memoryProperties);
239     }
240 
241     auto supportDualStorageSharedMemory = memoryManager->isLocalMemorySupported(*memoryProperties.rootDeviceIndices.begin());
242 
243     if (DebugManager.flags.AllocateSharedAllocationsWithCpuAndGpuStorage.get() != -1) {
244         supportDualStorageSharedMemory = !!DebugManager.flags.AllocateSharedAllocationsWithCpuAndGpuStorage.get();
245     }
246 
247     if (supportDualStorageSharedMemory) {
248         bool useKmdMigration = memoryManager->isKmdMigrationAvailable(*memoryProperties.rootDeviceIndices.begin());
249         void *unifiedMemoryPointer = nullptr;
250 
251         if (useKmdMigration) {
252             unifiedMemoryPointer = createUnifiedKmdMigratedAllocation(size, {}, memoryProperties);
253             if (!unifiedMemoryPointer) {
254                 return nullptr;
255             }
256         } else {
257             unifiedMemoryPointer = createUnifiedAllocationWithDeviceStorage(size, {}, memoryProperties);
258             if (!unifiedMemoryPointer) {
259                 return nullptr;
260             }
261 
262             UNRECOVERABLE_IF(cmdQ == nullptr);
263             auto pageFaultManager = this->memoryManager->getPageFaultManager();
264             pageFaultManager->insertAllocation(unifiedMemoryPointer, size, this, cmdQ, memoryProperties.allocationFlags);
265         }
266 
267         auto unifiedMemoryAllocation = this->getSVMAlloc(unifiedMemoryPointer);
268         unifiedMemoryAllocation->memoryType = memoryProperties.memoryType;
269         unifiedMemoryAllocation->allocationFlagsProperty = memoryProperties.allocationFlags;
270 
271         return unifiedMemoryPointer;
272     }
273     return createUnifiedMemoryAllocation(size, memoryProperties);
274 }
275 
createUnifiedKmdMigratedAllocation(size_t size,const SvmAllocationProperties & svmProperties,const UnifiedMemoryProperties & unifiedMemoryProperties)276 void *SVMAllocsManager::createUnifiedKmdMigratedAllocation(size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties) {
277 
278     auto rootDeviceIndex = unifiedMemoryProperties.device
279                                ? unifiedMemoryProperties.device->getRootDeviceIndex()
280                                : *unifiedMemoryProperties.rootDeviceIndices.begin();
281     auto &deviceBitfield = unifiedMemoryProperties.subdeviceBitfields.at(rootDeviceIndex);
282     size_t alignedSize = alignUp<size_t>(size, 2 * MemoryConstants::megaByte);
283     AllocationProperties gpuProperties{rootDeviceIndex,
284                                        true,
285                                        alignedSize,
286                                        GraphicsAllocation::AllocationType::UNIFIED_SHARED_MEMORY,
287                                        false,
288                                        false,
289                                        deviceBitfield};
290 
291     gpuProperties.alignment = 2 * MemoryConstants::megaByte;
292     auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags);
293     MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false, cacheRegion);
294     GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties);
295     if (!allocationGpu) {
296         return nullptr;
297     }
298     setUnifiedAllocationProperties(allocationGpu, svmProperties);
299 
300     SvmAllocationData allocData(rootDeviceIndex);
301     allocData.gpuAllocations.addAllocation(allocationGpu);
302     allocData.cpuAllocation = nullptr;
303     allocData.device = unifiedMemoryProperties.device;
304     allocData.size = size;
305     allocData.setAllocId(this->allocationsCounter++);
306 
307     std::unique_lock<SpinLock> lock(mtx);
308     this->SVMAllocs.insert(allocData);
309     return allocationGpu->getUnderlyingBuffer();
310 }
311 
setUnifiedAllocationProperties(GraphicsAllocation * allocation,const SvmAllocationProperties & svmProperties)312 void SVMAllocsManager::setUnifiedAllocationProperties(GraphicsAllocation *allocation, const SvmAllocationProperties &svmProperties) {
313     allocation->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly);
314     allocation->setCoherent(svmProperties.coherent);
315 }
316 
getSVMAlloc(const void * ptr)317 SvmAllocationData *SVMAllocsManager::getSVMAlloc(const void *ptr) {
318     std::unique_lock<SpinLock> lock(mtx);
319     return SVMAllocs.get(ptr);
320 }
321 
insertSVMAlloc(const SvmAllocationData & svmAllocData)322 void SVMAllocsManager::insertSVMAlloc(const SvmAllocationData &svmAllocData) {
323     std::unique_lock<SpinLock> lock(mtx);
324     SVMAllocs.insert(svmAllocData);
325 }
326 
removeSVMAlloc(const SvmAllocationData & svmAllocData)327 void SVMAllocsManager::removeSVMAlloc(const SvmAllocationData &svmAllocData) {
328     std::unique_lock<SpinLock> lock(mtx);
329     SVMAllocs.remove(svmAllocData);
330 }
331 
freeSVMAlloc(void * ptr,bool blocking)332 bool SVMAllocsManager::freeSVMAlloc(void *ptr, bool blocking) {
333     SvmAllocationData *svmData = getSVMAlloc(ptr);
334     if (svmData) {
335         if (blocking) {
336             if (svmData->cpuAllocation) {
337                 this->memoryManager->waitForEnginesCompletion(*svmData->cpuAllocation);
338             }
339 
340             for (auto &gpuAllocation : svmData->gpuAllocations.getGraphicsAllocations()) {
341                 if (gpuAllocation) {
342                     this->memoryManager->waitForEnginesCompletion(*gpuAllocation);
343                 }
344             }
345         }
346 
347         auto pageFaultManager = this->memoryManager->getPageFaultManager();
348         if (pageFaultManager) {
349             pageFaultManager->removeAllocation(ptr);
350         }
351         std::unique_lock<SpinLock> lock(mtx);
352         if (svmData->gpuAllocations.getAllocationType() == GraphicsAllocation::AllocationType::SVM_ZERO_COPY) {
353             freeZeroCopySvmAllocation(svmData);
354         } else {
355             freeSvmAllocationWithDeviceStorage(svmData);
356         }
357         return true;
358     }
359     return false;
360 }
361 
createZeroCopySvmAllocation(size_t size,const SvmAllocationProperties & svmProperties,const std::set<uint32_t> & rootDeviceIndices,const std::map<uint32_t,DeviceBitfield> & subdeviceBitfields)362 void *SVMAllocsManager::createZeroCopySvmAllocation(size_t size, const SvmAllocationProperties &svmProperties,
363                                                     const std::set<uint32_t> &rootDeviceIndices,
364                                                     const std::map<uint32_t, DeviceBitfield> &subdeviceBitfields) {
365 
366     auto rootDeviceIndex = *rootDeviceIndices.begin();
367     auto &deviceBitfield = subdeviceBitfields.at(rootDeviceIndex);
368     AllocationProperties properties{rootDeviceIndex,
369                                     true, // allocateMemory
370                                     size,
371                                     GraphicsAllocation::AllocationType::SVM_ZERO_COPY,
372                                     false, // isMultiStorageAllocation
373                                     deviceBitfield};
374     MemoryPropertiesHelper::fillCachePolicyInProperties(properties, false, svmProperties.readOnly, false, properties.cacheRegion);
375 
376     std::vector<uint32_t> rootDeviceIndicesVector(rootDeviceIndices.begin(), rootDeviceIndices.end());
377 
378     auto maxRootDeviceIndex = *std::max_element(rootDeviceIndices.begin(), rootDeviceIndices.end(), std::less<uint32_t const>());
379     SvmAllocationData allocData(maxRootDeviceIndex);
380 
381     void *usmPtr = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndicesVector, properties, allocData.gpuAllocations);
382     if (!usmPtr) {
383         return nullptr;
384     }
385     for (const auto &rootDeviceIndex : rootDeviceIndices) {
386         auto allocation = allocData.gpuAllocations.getGraphicsAllocation(rootDeviceIndex);
387         allocation->setMemObjectsAllocationWithWritableFlags(!svmProperties.readOnly && !svmProperties.hostPtrReadOnly);
388         allocation->setCoherent(svmProperties.coherent);
389     }
390     allocData.size = size;
391 
392     std::unique_lock<SpinLock> lock(mtx);
393     this->SVMAllocs.insert(allocData);
394     return usmPtr;
395 }
396 
createUnifiedAllocationWithDeviceStorage(size_t size,const SvmAllocationProperties & svmProperties,const UnifiedMemoryProperties & unifiedMemoryProperties)397 void *SVMAllocsManager::createUnifiedAllocationWithDeviceStorage(size_t size, const SvmAllocationProperties &svmProperties, const UnifiedMemoryProperties &unifiedMemoryProperties) {
398     auto rootDeviceIndex = unifiedMemoryProperties.device
399                                ? unifiedMemoryProperties.device->getRootDeviceIndex()
400                                : *unifiedMemoryProperties.rootDeviceIndices.begin();
401     size_t alignedSizeCpu = alignUp<size_t>(size, MemoryConstants::pageSize2Mb);
402     size_t alignedSizeGpu = alignUp<size_t>(size, MemoryConstants::pageSize64k);
403     DeviceBitfield subDevices = unifiedMemoryProperties.subdeviceBitfields.at(rootDeviceIndex);
404     AllocationProperties cpuProperties{rootDeviceIndex,
405                                        true, // allocateMemory
406                                        alignedSizeCpu, GraphicsAllocation::AllocationType::SVM_CPU,
407                                        false, // isMultiStorageAllocation
408                                        subDevices};
409     cpuProperties.alignment = MemoryConstants::pageSize2Mb;
410     auto cacheRegion = MemoryPropertiesHelper::getCacheRegion(unifiedMemoryProperties.allocationFlags);
411     MemoryPropertiesHelper::fillCachePolicyInProperties(cpuProperties, false, svmProperties.readOnly, false, cacheRegion);
412     GraphicsAllocation *allocationCpu = memoryManager->allocateGraphicsMemoryWithProperties(cpuProperties);
413     if (!allocationCpu) {
414         return nullptr;
415     }
416     setUnifiedAllocationProperties(allocationCpu, svmProperties);
417     void *svmPtr = allocationCpu->getUnderlyingBuffer();
418 
419     bool multiStorageAllocation = (subDevices.count() > 1) && multiOsContextSupport;
420     if ((subDevices.count() > 1) && !multiOsContextSupport) {
421         for (uint32_t i = 0;; i++) {
422             if (subDevices.test(i)) {
423                 subDevices.reset();
424                 subDevices.set(i);
425                 break;
426             }
427         }
428     }
429 
430     AllocationProperties gpuProperties{rootDeviceIndex,
431                                        false,
432                                        alignedSizeGpu,
433                                        GraphicsAllocation::AllocationType::SVM_GPU,
434                                        false,
435                                        multiStorageAllocation,
436                                        subDevices};
437 
438     gpuProperties.alignment = MemoryConstants::pageSize64k;
439     MemoryPropertiesHelper::fillCachePolicyInProperties(gpuProperties, false, svmProperties.readOnly, false, cacheRegion);
440     GraphicsAllocation *allocationGpu = memoryManager->allocateGraphicsMemoryWithProperties(gpuProperties, svmPtr);
441     if (!allocationGpu) {
442         memoryManager->freeGraphicsMemory(allocationCpu);
443         return nullptr;
444     }
445     setUnifiedAllocationProperties(allocationGpu, svmProperties);
446 
447     SvmAllocationData allocData(rootDeviceIndex);
448     allocData.gpuAllocations.addAllocation(allocationGpu);
449     allocData.cpuAllocation = allocationCpu;
450     allocData.device = unifiedMemoryProperties.device;
451     allocData.size = size;
452     allocData.setAllocId(this->allocationsCounter++);
453 
454     std::unique_lock<SpinLock> lock(mtx);
455     this->SVMAllocs.insert(allocData);
456     return svmPtr;
457 }
458 
freeZeroCopySvmAllocation(SvmAllocationData * svmData)459 void SVMAllocsManager::freeZeroCopySvmAllocation(SvmAllocationData *svmData) {
460     auto gpuAllocations = svmData->gpuAllocations;
461     SVMAllocs.remove(*svmData);
462     for (const auto &graphicsAllocation : gpuAllocations.getGraphicsAllocations()) {
463         memoryManager->freeGraphicsMemory(graphicsAllocation);
464     }
465 }
466 
freeSvmAllocationWithDeviceStorage(SvmAllocationData * svmData)467 void SVMAllocsManager::freeSvmAllocationWithDeviceStorage(SvmAllocationData *svmData) {
468     auto graphicsAllocations = svmData->gpuAllocations.getGraphicsAllocations();
469     GraphicsAllocation *cpuAllocation = svmData->cpuAllocation;
470     SVMAllocs.remove(*svmData);
471 
472     for (auto gpuAllocation : graphicsAllocations) {
473         memoryManager->freeGraphicsMemory(gpuAllocation);
474     }
475     memoryManager->freeGraphicsMemory(cpuAllocation);
476 }
477 
hasHostAllocations()478 bool SVMAllocsManager::hasHostAllocations() {
479     std::unique_lock<SpinLock> lock(mtx);
480     for (auto &allocation : this->SVMAllocs.allocations) {
481         if (allocation.second.memoryType == InternalMemoryType::HOST_UNIFIED_MEMORY) {
482             return true;
483         }
484     }
485     return false;
486 }
487 
getSvmMapOperation(const void * ptr)488 SvmMapOperation *SVMAllocsManager::getSvmMapOperation(const void *ptr) {
489     std::unique_lock<SpinLock> lock(mtx);
490     return svmMapOperations.get(ptr);
491 }
492 
insertSvmMapOperation(void * regionSvmPtr,size_t regionSize,void * baseSvmPtr,size_t offset,bool readOnlyMap)493 void SVMAllocsManager::insertSvmMapOperation(void *regionSvmPtr, size_t regionSize, void *baseSvmPtr, size_t offset, bool readOnlyMap) {
494     SvmMapOperation svmMapOperation;
495     svmMapOperation.regionSvmPtr = regionSvmPtr;
496     svmMapOperation.baseSvmPtr = baseSvmPtr;
497     svmMapOperation.offset = offset;
498     svmMapOperation.regionSize = regionSize;
499     svmMapOperation.readOnlyMap = readOnlyMap;
500     std::unique_lock<SpinLock> lock(mtx);
501     svmMapOperations.insert(svmMapOperation);
502 }
503 
removeSvmMapOperation(const void * regionSvmPtr)504 void SVMAllocsManager::removeSvmMapOperation(const void *regionSvmPtr) {
505     std::unique_lock<SpinLock> lock(mtx);
506     svmMapOperations.remove(regionSvmPtr);
507 }
508 
getGraphicsAllocationTypeAndCompressionPreference(const UnifiedMemoryProperties & unifiedMemoryProperties,bool & compressionEnabled) const509 GraphicsAllocation::AllocationType SVMAllocsManager::getGraphicsAllocationTypeAndCompressionPreference(const UnifiedMemoryProperties &unifiedMemoryProperties, bool &compressionEnabled) const {
510     compressionEnabled = false;
511 
512     GraphicsAllocation::AllocationType allocationType = GraphicsAllocation::AllocationType::BUFFER_HOST_MEMORY;
513     if (unifiedMemoryProperties.memoryType == InternalMemoryType::DEVICE_UNIFIED_MEMORY) {
514         if (unifiedMemoryProperties.allocationFlags.allocFlags.allocWriteCombined) {
515             allocationType = GraphicsAllocation::AllocationType::WRITE_COMBINED;
516         } else {
517             UNRECOVERABLE_IF(nullptr == unifiedMemoryProperties.device);
518             const auto &hwInfoConfig = *HwInfoConfig::get(unifiedMemoryProperties.device->getHardwareInfo().platform.eProductFamily);
519             if (hwInfoConfig.allowStatelessCompression(unifiedMemoryProperties.device->getHardwareInfo())) {
520                 compressionEnabled = true;
521             }
522             allocationType = GraphicsAllocation::AllocationType::BUFFER;
523         }
524     }
525     return allocationType;
526 }
527 
528 } // namespace NEO
529