1 /*
2  * Copyright (C) 2018-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #pragma once
9 #include "shared/source/memory_manager/memory_manager.h"
10 #include "shared/source/os_interface/linux/drm_allocation.h"
11 #include "shared/source/os_interface/linux/drm_buffer_object.h"
12 #include "shared/source/os_interface/linux/drm_neo.h"
13 
14 #include "drm_gem_close_worker.h"
15 
16 #include <limits>
17 #include <map>
18 #include <sys/mman.h>
19 
20 namespace NEO {
21 class BufferObject;
22 class Drm;
23 
24 class DrmMemoryManager : public MemoryManager {
25   public:
26     DrmMemoryManager(gemCloseWorkerMode mode,
27                      bool forcePinAllowed,
28                      bool validateHostPtrMemory,
29                      ExecutionEnvironment &executionEnvironment);
30     ~DrmMemoryManager() override;
31 
32     void initialize(gemCloseWorkerMode mode);
33     void addAllocationToHostPtrManager(GraphicsAllocation *gfxAllocation) override;
34     void removeAllocationFromHostPtrManager(GraphicsAllocation *gfxAllocation) override;
35     void freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) override;
36     void handleFenceCompletion(GraphicsAllocation *allocation) override;
37     GraphicsAllocation *createGraphicsAllocationFromExistingStorage(AllocationProperties &properties, void *ptr, MultiGraphicsAllocation &multiGraphicsAllocation) override;
38     GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation) override;
39     void closeSharedHandle(GraphicsAllocation *gfxAllocation) override;
40     GraphicsAllocation *createPaddedAllocation(GraphicsAllocation *inputGraphicsAllocation, size_t sizeWithPadding) override;
createGraphicsAllocationFromNTHandle(void * handle,uint32_t rootDeviceIndex,GraphicsAllocation::AllocationType allocType)41     GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, GraphicsAllocation::AllocationType allocType) override { return nullptr; }
42 
43     uint64_t getSystemSharedMemory(uint32_t rootDeviceIndex) override;
44     uint64_t getLocalMemorySize(uint32_t rootDeviceIndex, uint32_t deviceBitfield) override;
45     double getPercentOfGlobalMemoryAvailable(uint32_t rootDeviceIndex) override;
46 
47     AllocationStatus populateOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
48     void cleanOsHandles(OsHandleStorage &handleStorage, uint32_t rootDeviceIndex) override;
49     void commonCleanup() override;
50 
51     // drm/i915 ioctl wrappers
52     MOCKABLE_VIRTUAL uint32_t unreference(BufferObject *bo, bool synchronousDestroy);
53 
isValidateHostMemoryEnabled()54     bool isValidateHostMemoryEnabled() const {
55         return validateHostPtrMemory;
56     }
57 
peekGemCloseWorker()58     DrmGemCloseWorker *peekGemCloseWorker() const { return this->gemCloseWorker.get(); }
59     bool copyMemoryToAllocation(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy) override;
60     bool copyMemoryToAllocationBanks(GraphicsAllocation *graphicsAllocation, size_t destinationOffset, const void *memoryToCopy, size_t sizeToCopy, DeviceBitfield handleMask) override;
61 
62     MOCKABLE_VIRTUAL int obtainFdFromHandle(int boHandle, uint32_t rootDeviceindex);
63     AddressRange reserveGpuAddress(size_t size, uint32_t rootDeviceIndex) override;
64     void freeGpuAddress(AddressRange addressRange, uint32_t rootDeviceIndex) override;
65     MOCKABLE_VIRTUAL BufferObject *createBufferObjectInMemoryRegion(Drm *drm, uint64_t gpuAddress, size_t size, uint32_t memoryBanks, size_t maxOsContextCount);
66 
67     bool isKmdMigrationAvailable(uint32_t rootDeviceIndex) override;
68 
69     bool setMemAdvise(GraphicsAllocation *gfxAllocation, MemAdviseFlags flags, uint32_t rootDeviceIndex) override;
70 
71     std::unique_lock<std::mutex> acquireAllocLock();
72     std::vector<GraphicsAllocation *> &getSysMemAllocs();
73     std::vector<GraphicsAllocation *> &getLocalMemAllocs(uint32_t rootDeviceIndex);
74     void registerSysMemAlloc(GraphicsAllocation *allocation) override;
75     void registerLocalMemAlloc(GraphicsAllocation *allocation, uint32_t rootDeviceIndex) override;
76     void unregisterAllocation(GraphicsAllocation *allocation);
77 
78     static std::unique_ptr<MemoryManager> create(ExecutionEnvironment &executionEnvironment);
79 
80     DrmAllocation *createUSMHostAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool hasMappedPtr);
81     void releaseDeviceSpecificMemResources(uint32_t rootDeviceIndex) override;
82     void createDeviceSpecificMemResources(uint32_t rootDeviceIndex) override;
83 
84   protected:
85     BufferObject *findAndReferenceSharedBufferObject(int boHandle, uint32_t rootDeviceIndex);
86     void eraseSharedBufferObject(BufferObject *bo);
87     void pushSharedBufferObject(BufferObject *bo);
88     BufferObject *allocUserptr(uintptr_t address, size_t size, uint64_t flags, uint32_t rootDeviceIndex);
89     bool setDomainCpu(GraphicsAllocation &graphicsAllocation, bool writeEnable);
90     uint64_t acquireGpuRange(size_t &size, uint32_t rootDeviceIndex, HeapIndex heapIndex);
91     MOCKABLE_VIRTUAL void releaseGpuRange(void *address, size_t size, uint32_t rootDeviceIndex);
92     void emitPinningRequest(BufferObject *bo, const AllocationData &allocationData) const;
93     uint32_t getDefaultDrmContextId(uint32_t rootDeviceIndex) const;
94     size_t getUserptrAlignment();
95 
96     DrmAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override;
97     DrmAllocation *allocateGraphicsMemoryForNonSvmHostPtr(const AllocationData &allocationData) override;
98     DrmAllocation *allocateGraphicsMemoryWithAlignment(const AllocationData &allocationData) override;
99     DrmAllocation *allocateGraphicsMemoryWithAlignmentImpl(const AllocationData &allocationData);
100     DrmAllocation *createAllocWithAlignmentFromUserptr(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSVMSize, uint64_t gpuAddress);
101     DrmAllocation *createAllocWithAlignment(const AllocationData &allocationData, size_t size, size_t alignment, size_t alignedSize, uint64_t gpuAddress);
102     DrmAllocation *createMultiHostAllocation(const AllocationData &allocationData);
103     void obtainGpuAddress(const AllocationData &allocationData, BufferObject *bo, uint64_t gpuAddress);
104     DrmAllocation *allocateUSMHostGraphicsMemory(const AllocationData &allocationData) override;
105     DrmAllocation *allocateGraphicsMemoryWithHostPtr(const AllocationData &allocationData) override;
106     DrmAllocation *allocateGraphicsMemory64kb(const AllocationData &allocationData) override;
107     GraphicsAllocation *allocateMemoryByKMD(const AllocationData &allocationData) override;
108     GraphicsAllocation *allocateGraphicsMemoryForImageImpl(const AllocationData &allocationData, std::unique_ptr<Gmm> gmm) override;
109     GraphicsAllocation *allocateGraphicsMemoryWithGpuVa(const AllocationData &allocationData) override;
110     GraphicsAllocation *createSharedUnifiedMemoryAllocation(const AllocationData &allocationData);
111 
112     void *lockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
113     void *lockResourceInLocalMemoryImpl(GraphicsAllocation &graphicsAllocation);
114     MOCKABLE_VIRTUAL void *lockResourceInLocalMemoryImpl(BufferObject *bo);
115     MOCKABLE_VIRTUAL void unlockResourceInLocalMemoryImpl(BufferObject *bo);
116     void unlockResourceImpl(GraphicsAllocation &graphicsAllocation) override;
117     DrmAllocation *allocate32BitGraphicsMemoryImpl(const AllocationData &allocationData, bool useLocalMemory) override;
118     GraphicsAllocation *allocateGraphicsMemoryInDevicePool(const AllocationData &allocationData, AllocationStatus &status) override;
119     bool createDrmAllocation(Drm *drm, DrmAllocation *allocation, uint64_t gpuAddress, size_t maxOsContextCount);
120     void registerAllocationInOs(GraphicsAllocation *allocation) override;
121 
122     Drm &getDrm(uint32_t rootDeviceIndex) const;
123     uint32_t getRootDeviceIndex(const Drm *drm);
124     BufferObject *createRootDeviceBufferObject(uint32_t rootDeviceIndex);
125     void releaseBufferObject(uint32_t rootDeviceIndex);
126     bool retrieveMmapOffsetForBufferObject(uint32_t rootDeviceIndex, BufferObject &bo, uint64_t flags, uint64_t &offset);
127 
128     std::vector<BufferObject *> pinBBs;
129     std::vector<void *> memoryForPinBBs;
130     size_t pinThreshold = 8 * 1024 * 1024;
131     bool forcePinEnabled = false;
132     const bool validateHostPtrMemory;
133     std::unique_ptr<DrmGemCloseWorker> gemCloseWorker;
134     decltype(&mmap) mmapFunction = mmap;
135     decltype(&munmap) munmapFunction = munmap;
136     decltype(&lseek) lseekFunction = lseek;
137     decltype(&close) closeFunction = close;
138     std::vector<BufferObject *> sharingBufferObjects;
139     std::mutex mtx;
140 
141     std::vector<std::vector<GraphicsAllocation *>> localMemAllocs;
142     std::vector<GraphicsAllocation *> sysMemAllocs;
143     std::mutex allocMutex;
144 };
145 } // namespace NEO
146