1 /*
2  * Copyright (C) 2018-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #pragma once
9 #include "shared/source/helpers/basic_math.h"
10 #include "shared/source/helpers/constants.h"
11 
12 #include "opencl/extensions/public/cl_ext_private.h"
13 #include "opencl/source/context/context_type.h"
14 #include "opencl/source/mem_obj/mem_obj.h"
15 
16 #include "igfxfmid.h"
17 #include "memory_properties_flags.h"
18 
19 #include <functional>
20 
21 namespace NEO {
22 class Buffer;
23 class ClDevice;
24 class Device;
25 class MemoryManager;
26 struct EncodeSurfaceStateArgs;
27 
28 using BufferCreatFunc = Buffer *(*)(Context *context,
29                                     MemoryProperties memoryProperties,
30                                     cl_mem_flags flags,
31                                     cl_mem_flags_intel flagsIntel,
32                                     size_t size,
33                                     void *memoryStorage,
34                                     void *hostPtr,
35                                     MultiGraphicsAllocation multiGraphicsAllocation,
36                                     bool zeroCopy,
37                                     bool isHostPtrSVM,
38                                     bool isImageRedescribed);
39 
40 struct BufferFactoryFuncs {
41     BufferCreatFunc createBufferFunction;
42 };
43 
44 extern BufferFactoryFuncs bufferFactory[IGFX_MAX_CORE];
45 
46 namespace BufferFunctions {
47 using ValidateInputAndCreateBufferFunc = std::function<cl_mem(cl_context context,
48                                                               const uint64_t *properties,
49                                                               uint64_t flags,
50                                                               uint64_t flagsIntel,
51                                                               size_t size,
52                                                               void *hostPtr,
53                                                               int32_t &retVal)>;
54 extern ValidateInputAndCreateBufferFunc validateInputAndCreateBuffer;
55 } // namespace BufferFunctions
56 
57 class Buffer : public MemObj {
58   public:
59     constexpr static size_t maxBufferSizeForReadWriteOnCpu = 10 * MB;
60     constexpr static cl_ulong maskMagic = 0xFFFFFFFFFFFFFFFFLL;
61     constexpr static cl_ulong objectMagic = MemObj::objectMagic | 0x02;
62     bool forceDisallowCPUCopy = false;
63 
64     ~Buffer() override;
65 
66     static cl_mem validateInputAndCreateBuffer(cl_context context,
67                                                const cl_mem_properties *properties,
68                                                cl_mem_flags flags,
69                                                cl_mem_flags_intel flagsIntel,
70                                                size_t size,
71                                                void *hostPtr,
72                                                cl_int &retVal);
73 
74     static Buffer *create(Context *context,
75                           cl_mem_flags flags,
76                           size_t size,
77                           void *hostPtr,
78                           cl_int &errcodeRet);
79 
80     static Buffer *create(Context *context,
81                           MemoryProperties properties,
82                           cl_mem_flags flags,
83                           cl_mem_flags_intel flagsIntel,
84                           size_t size,
85                           void *hostPtr,
86                           cl_int &errcodeRet);
87 
88     static Buffer *createSharedBuffer(Context *context,
89                                       cl_mem_flags flags,
90                                       SharingHandler *sharingHandler,
91                                       MultiGraphicsAllocation multiGraphicsAllocation);
92 
93     static Buffer *createBufferHw(Context *context,
94                                   MemoryProperties memoryProperties,
95                                   cl_mem_flags flags,
96                                   cl_mem_flags_intel flagsIntel,
97                                   size_t size,
98                                   void *memoryStorage,
99                                   void *hostPtr,
100                                   MultiGraphicsAllocation multiGraphicsAllocation,
101                                   bool zeroCopy,
102                                   bool isHostPtrSVM,
103                                   bool isImageRedescribed);
104 
105     static Buffer *createBufferHwFromDevice(const Device *device,
106                                             cl_mem_flags flags,
107                                             cl_mem_flags_intel flagsIntel,
108                                             size_t size,
109                                             void *memoryStorage,
110                                             void *hostPtr,
111                                             MultiGraphicsAllocation multiGraphicsAllocation,
112                                             size_t offset,
113                                             bool zeroCopy,
114                                             bool isHostPtrSVM,
115                                             bool isImageRedescribed);
116 
117     Buffer *createSubBuffer(cl_mem_flags flags,
118                             cl_mem_flags_intel flagsIntel,
119                             const cl_buffer_region *region,
120                             cl_int &errcodeRet);
121 
122     static void setSurfaceState(const Device *device,
123                                 void *surfaceState,
124                                 bool forceNonAuxMode,
125                                 bool disableL3,
126                                 size_t svmSize,
127                                 void *svmPtr,
128                                 size_t offset,
129                                 GraphicsAllocation *gfxAlloc,
130                                 cl_mem_flags flags,
131                                 cl_mem_flags_intel flagsIntel,
132                                 bool useGlobalAtomics,
133                                 bool areMultipleSubDevicesInContext);
134 
135     static void provideCompressionHint(bool compressionEnabled, Context *context, Buffer *buffer);
136 
137     BufferCreatFunc createFunction = nullptr;
138     bool isSubBuffer();
139     bool isValidSubBufferOffset(size_t offset);
140     uint64_t setArgStateless(void *memory, uint32_t patchSize, uint32_t rootDeviceIndex, bool set32BitAddressing);
141     virtual void setArgStateful(void *memory, bool forceNonAuxMode, bool disableL3, bool alignSizeForAuxTranslation,
142                                 bool isReadOnly, const Device &device, bool useGlobalAtomics, bool areMultipleSubDevicesInContext) = 0;
143     bool bufferRectPitchSet(const size_t *bufferOrigin,
144                             const size_t *region,
145                             size_t &bufferRowPitch,
146                             size_t &bufferSlicePitch,
147                             size_t &hostRowPitch,
148                             size_t &hostSlicePitch,
149                             bool isSrcBuffer);
150 
151     static size_t calculateHostPtrSize(const size_t *origin, const size_t *region, size_t rowPitch, size_t slicePitch);
152 
153     void transferDataToHostPtr(MemObjSizeArray &copySize, MemObjOffsetArray &copyOffset) override;
154     void transferDataFromHostPtr(MemObjSizeArray &copySize, MemObjOffsetArray &copyOffset) override;
155 
156     bool isReadWriteOnCpuAllowed(const Device &device);
157     bool isReadWriteOnCpuPreferred(void *ptr, size_t size, const Device &device);
158 
159     uint32_t getMocsValue(bool disableL3Cache, bool isReadOnlyArgument, uint32_t rootDeviceIndex) const;
160     uint32_t getSurfaceSize(bool alignSizeForAuxTranslation, uint32_t rootDeviceIndex) const;
161     uint64_t getBufferAddress(uint32_t rootDeviceIndex) const;
162 
163     bool isCompressed(uint32_t rootDeviceIndex) const;
164 
165   protected:
166     Buffer(Context *context,
167            MemoryProperties memoryProperties,
168            cl_mem_flags flags,
169            cl_mem_flags_intel flagsIntel,
170            size_t size,
171            void *memoryStorage,
172            void *hostPtr,
173            MultiGraphicsAllocation multiGraphicsAllocation,
174            bool zeroCopy,
175            bool isHostPtrSVM,
176            bool isObjectRedescribed);
177 
178     Buffer();
179 
180     static void checkMemory(MemoryProperties memoryProperties,
181                             size_t size,
182                             void *hostPtr,
183                             cl_int &errcodeRet,
184                             bool &isZeroCopy,
185                             bool &copyMemoryFromHostPtr,
186                             MemoryManager *memMngr,
187                             uint32_t rootDeviceIndex,
188                             bool forceCopyHostPtr);
189     static GraphicsAllocation::AllocationType getGraphicsAllocationTypeAndCompressionPreference(const MemoryProperties &properties, Context &context,
190                                                                                                 bool &compressionEnabled, bool localMemoryEnabled);
191     static bool isReadOnlyMemoryPermittedByFlags(const MemoryProperties &properties);
192 
193     void transferData(void *dst, void *src, size_t copySize, size_t copyOffset);
194 
195     void appendSurfaceStateArgs(EncodeSurfaceStateArgs &args);
196 };
197 
198 template <typename GfxFamily>
199 class BufferHw : public Buffer {
200   public:
BufferHw(Context * context,MemoryProperties memoryProperties,cl_mem_flags flags,cl_mem_flags_intel flagsIntel,size_t size,void * memoryStorage,void * hostPtr,MultiGraphicsAllocation multiGraphicsAllocation,bool zeroCopy,bool isHostPtrSVM,bool isObjectRedescribed)201     BufferHw(Context *context,
202              MemoryProperties memoryProperties,
203              cl_mem_flags flags,
204              cl_mem_flags_intel flagsIntel,
205              size_t size,
206              void *memoryStorage,
207              void *hostPtr,
208              MultiGraphicsAllocation multiGraphicsAllocation,
209              bool zeroCopy,
210              bool isHostPtrSVM,
211              bool isObjectRedescribed)
212         : Buffer(context, memoryProperties, flags, flagsIntel, size, memoryStorage, hostPtr, std::move(multiGraphicsAllocation),
213                  zeroCopy, isHostPtrSVM, isObjectRedescribed) {}
214 
215     void setArgStateful(void *memory, bool forceNonAuxMode, bool disableL3, bool alignSizeForAuxTranslation,
216                         bool isReadOnlyArgument, const Device &device, bool useGlobalAtomics, bool areMultipleSubDevicesInContext) override;
217 
create(Context * context,MemoryProperties memoryProperties,cl_mem_flags flags,cl_mem_flags_intel flagsIntel,size_t size,void * memoryStorage,void * hostPtr,MultiGraphicsAllocation multiGraphicsAllocation,bool zeroCopy,bool isHostPtrSVM,bool isObjectRedescribed)218     static Buffer *create(Context *context,
219                           MemoryProperties memoryProperties,
220                           cl_mem_flags flags,
221                           cl_mem_flags_intel flagsIntel,
222                           size_t size,
223                           void *memoryStorage,
224                           void *hostPtr,
225                           MultiGraphicsAllocation multiGraphicsAllocation,
226                           bool zeroCopy,
227                           bool isHostPtrSVM,
228                           bool isObjectRedescribed) {
229         auto buffer = new BufferHw<GfxFamily>(context,
230                                               memoryProperties,
231                                               flags,
232                                               flagsIntel,
233                                               size,
234                                               memoryStorage,
235                                               hostPtr,
236                                               std::move(multiGraphicsAllocation),
237                                               zeroCopy,
238                                               isHostPtrSVM,
239                                               isObjectRedescribed);
240         buffer->surfaceType = SURFACE_STATE::SURFACE_TYPE_SURFTYPE_1D;
241         return buffer;
242     }
243 
244     typedef typename GfxFamily::RENDER_SURFACE_STATE SURFACE_STATE;
245     typename SURFACE_STATE::SURFACE_TYPE surfaceType;
246 };
247 
248 } // namespace NEO
249