1 /* 2 * Copyright (C) 2018-2021 Intel Corporation 3 * 4 * SPDX-License-Identifier: MIT 5 * 6 */ 7 8 #pragma once 9 #include "shared/source/helpers/basic_math.h" 10 #include "shared/source/helpers/constants.h" 11 12 #include "opencl/extensions/public/cl_ext_private.h" 13 #include "opencl/source/context/context_type.h" 14 #include "opencl/source/mem_obj/mem_obj.h" 15 16 #include "igfxfmid.h" 17 #include "memory_properties_flags.h" 18 19 #include <functional> 20 21 namespace NEO { 22 class Buffer; 23 class ClDevice; 24 class Device; 25 class MemoryManager; 26 struct EncodeSurfaceStateArgs; 27 28 using BufferCreatFunc = Buffer *(*)(Context *context, 29 MemoryProperties memoryProperties, 30 cl_mem_flags flags, 31 cl_mem_flags_intel flagsIntel, 32 size_t size, 33 void *memoryStorage, 34 void *hostPtr, 35 MultiGraphicsAllocation multiGraphicsAllocation, 36 bool zeroCopy, 37 bool isHostPtrSVM, 38 bool isImageRedescribed); 39 40 struct BufferFactoryFuncs { 41 BufferCreatFunc createBufferFunction; 42 }; 43 44 extern BufferFactoryFuncs bufferFactory[IGFX_MAX_CORE]; 45 46 namespace BufferFunctions { 47 using ValidateInputAndCreateBufferFunc = std::function<cl_mem(cl_context context, 48 const uint64_t *properties, 49 uint64_t flags, 50 uint64_t flagsIntel, 51 size_t size, 52 void *hostPtr, 53 int32_t &retVal)>; 54 extern ValidateInputAndCreateBufferFunc validateInputAndCreateBuffer; 55 } // namespace BufferFunctions 56 57 class Buffer : public MemObj { 58 public: 59 constexpr static size_t maxBufferSizeForReadWriteOnCpu = 10 * MB; 60 constexpr static cl_ulong maskMagic = 0xFFFFFFFFFFFFFFFFLL; 61 constexpr static cl_ulong objectMagic = MemObj::objectMagic | 0x02; 62 bool forceDisallowCPUCopy = false; 63 64 ~Buffer() override; 65 66 static cl_mem validateInputAndCreateBuffer(cl_context context, 67 const cl_mem_properties *properties, 68 cl_mem_flags flags, 69 cl_mem_flags_intel flagsIntel, 70 size_t size, 71 void *hostPtr, 72 cl_int &retVal); 73 74 static Buffer *create(Context *context, 75 cl_mem_flags flags, 76 size_t size, 77 void *hostPtr, 78 cl_int &errcodeRet); 79 80 static Buffer *create(Context *context, 81 MemoryProperties properties, 82 cl_mem_flags flags, 83 cl_mem_flags_intel flagsIntel, 84 size_t size, 85 void *hostPtr, 86 cl_int &errcodeRet); 87 88 static Buffer *createSharedBuffer(Context *context, 89 cl_mem_flags flags, 90 SharingHandler *sharingHandler, 91 MultiGraphicsAllocation multiGraphicsAllocation); 92 93 static Buffer *createBufferHw(Context *context, 94 MemoryProperties memoryProperties, 95 cl_mem_flags flags, 96 cl_mem_flags_intel flagsIntel, 97 size_t size, 98 void *memoryStorage, 99 void *hostPtr, 100 MultiGraphicsAllocation multiGraphicsAllocation, 101 bool zeroCopy, 102 bool isHostPtrSVM, 103 bool isImageRedescribed); 104 105 static Buffer *createBufferHwFromDevice(const Device *device, 106 cl_mem_flags flags, 107 cl_mem_flags_intel flagsIntel, 108 size_t size, 109 void *memoryStorage, 110 void *hostPtr, 111 MultiGraphicsAllocation multiGraphicsAllocation, 112 size_t offset, 113 bool zeroCopy, 114 bool isHostPtrSVM, 115 bool isImageRedescribed); 116 117 Buffer *createSubBuffer(cl_mem_flags flags, 118 cl_mem_flags_intel flagsIntel, 119 const cl_buffer_region *region, 120 cl_int &errcodeRet); 121 122 static void setSurfaceState(const Device *device, 123 void *surfaceState, 124 bool forceNonAuxMode, 125 bool disableL3, 126 size_t svmSize, 127 void *svmPtr, 128 size_t offset, 129 GraphicsAllocation *gfxAlloc, 130 cl_mem_flags flags, 131 cl_mem_flags_intel flagsIntel, 132 bool useGlobalAtomics, 133 bool areMultipleSubDevicesInContext); 134 135 static void provideCompressionHint(bool compressionEnabled, Context *context, Buffer *buffer); 136 137 BufferCreatFunc createFunction = nullptr; 138 bool isSubBuffer(); 139 bool isValidSubBufferOffset(size_t offset); 140 uint64_t setArgStateless(void *memory, uint32_t patchSize, uint32_t rootDeviceIndex, bool set32BitAddressing); 141 virtual void setArgStateful(void *memory, bool forceNonAuxMode, bool disableL3, bool alignSizeForAuxTranslation, 142 bool isReadOnly, const Device &device, bool useGlobalAtomics, bool areMultipleSubDevicesInContext) = 0; 143 bool bufferRectPitchSet(const size_t *bufferOrigin, 144 const size_t *region, 145 size_t &bufferRowPitch, 146 size_t &bufferSlicePitch, 147 size_t &hostRowPitch, 148 size_t &hostSlicePitch, 149 bool isSrcBuffer); 150 151 static size_t calculateHostPtrSize(const size_t *origin, const size_t *region, size_t rowPitch, size_t slicePitch); 152 153 void transferDataToHostPtr(MemObjSizeArray ©Size, MemObjOffsetArray ©Offset) override; 154 void transferDataFromHostPtr(MemObjSizeArray ©Size, MemObjOffsetArray ©Offset) override; 155 156 bool isReadWriteOnCpuAllowed(const Device &device); 157 bool isReadWriteOnCpuPreferred(void *ptr, size_t size, const Device &device); 158 159 uint32_t getMocsValue(bool disableL3Cache, bool isReadOnlyArgument, uint32_t rootDeviceIndex) const; 160 uint32_t getSurfaceSize(bool alignSizeForAuxTranslation, uint32_t rootDeviceIndex) const; 161 uint64_t getBufferAddress(uint32_t rootDeviceIndex) const; 162 163 bool isCompressed(uint32_t rootDeviceIndex) const; 164 165 protected: 166 Buffer(Context *context, 167 MemoryProperties memoryProperties, 168 cl_mem_flags flags, 169 cl_mem_flags_intel flagsIntel, 170 size_t size, 171 void *memoryStorage, 172 void *hostPtr, 173 MultiGraphicsAllocation multiGraphicsAllocation, 174 bool zeroCopy, 175 bool isHostPtrSVM, 176 bool isObjectRedescribed); 177 178 Buffer(); 179 180 static void checkMemory(MemoryProperties memoryProperties, 181 size_t size, 182 void *hostPtr, 183 cl_int &errcodeRet, 184 bool &isZeroCopy, 185 bool ©MemoryFromHostPtr, 186 MemoryManager *memMngr, 187 uint32_t rootDeviceIndex, 188 bool forceCopyHostPtr); 189 static GraphicsAllocation::AllocationType getGraphicsAllocationTypeAndCompressionPreference(const MemoryProperties &properties, Context &context, 190 bool &compressionEnabled, bool localMemoryEnabled); 191 static bool isReadOnlyMemoryPermittedByFlags(const MemoryProperties &properties); 192 193 void transferData(void *dst, void *src, size_t copySize, size_t copyOffset); 194 195 void appendSurfaceStateArgs(EncodeSurfaceStateArgs &args); 196 }; 197 198 template <typename GfxFamily> 199 class BufferHw : public Buffer { 200 public: BufferHw(Context * context,MemoryProperties memoryProperties,cl_mem_flags flags,cl_mem_flags_intel flagsIntel,size_t size,void * memoryStorage,void * hostPtr,MultiGraphicsAllocation multiGraphicsAllocation,bool zeroCopy,bool isHostPtrSVM,bool isObjectRedescribed)201 BufferHw(Context *context, 202 MemoryProperties memoryProperties, 203 cl_mem_flags flags, 204 cl_mem_flags_intel flagsIntel, 205 size_t size, 206 void *memoryStorage, 207 void *hostPtr, 208 MultiGraphicsAllocation multiGraphicsAllocation, 209 bool zeroCopy, 210 bool isHostPtrSVM, 211 bool isObjectRedescribed) 212 : Buffer(context, memoryProperties, flags, flagsIntel, size, memoryStorage, hostPtr, std::move(multiGraphicsAllocation), 213 zeroCopy, isHostPtrSVM, isObjectRedescribed) {} 214 215 void setArgStateful(void *memory, bool forceNonAuxMode, bool disableL3, bool alignSizeForAuxTranslation, 216 bool isReadOnlyArgument, const Device &device, bool useGlobalAtomics, bool areMultipleSubDevicesInContext) override; 217 create(Context * context,MemoryProperties memoryProperties,cl_mem_flags flags,cl_mem_flags_intel flagsIntel,size_t size,void * memoryStorage,void * hostPtr,MultiGraphicsAllocation multiGraphicsAllocation,bool zeroCopy,bool isHostPtrSVM,bool isObjectRedescribed)218 static Buffer *create(Context *context, 219 MemoryProperties memoryProperties, 220 cl_mem_flags flags, 221 cl_mem_flags_intel flagsIntel, 222 size_t size, 223 void *memoryStorage, 224 void *hostPtr, 225 MultiGraphicsAllocation multiGraphicsAllocation, 226 bool zeroCopy, 227 bool isHostPtrSVM, 228 bool isObjectRedescribed) { 229 auto buffer = new BufferHw<GfxFamily>(context, 230 memoryProperties, 231 flags, 232 flagsIntel, 233 size, 234 memoryStorage, 235 hostPtr, 236 std::move(multiGraphicsAllocation), 237 zeroCopy, 238 isHostPtrSVM, 239 isObjectRedescribed); 240 buffer->surfaceType = SURFACE_STATE::SURFACE_TYPE_SURFTYPE_1D; 241 return buffer; 242 } 243 244 typedef typename GfxFamily::RENDER_SURFACE_STATE SURFACE_STATE; 245 typename SURFACE_STATE::SURFACE_TYPE surfaceType; 246 }; 247 248 } // namespace NEO 249