1 /* 2 * Copyright (C) 2018-2021 Intel Corporation 3 * 4 * SPDX-License-Identifier: MIT 5 * 6 */ 7 8 #pragma once 9 #include "shared/source/helpers/string.h" 10 #include "shared/source/image/image_surface_state.h" 11 12 #include "opencl/source/helpers/cl_validators.h" 13 #include "opencl/source/helpers/surface_formats.h" 14 #include "opencl/source/mem_obj/buffer.h" 15 #include "opencl/source/mem_obj/mem_obj.h" 16 17 namespace NEO { 18 class Image; 19 struct KernelInfo; 20 struct SurfaceFormatInfo; 21 22 using ImageCreatFunc = Image *(*)(Context *context, 23 const MemoryProperties &memoryProperties, 24 uint64_t flags, 25 uint64_t flagsIntel, 26 size_t size, 27 void *hostPtr, 28 const cl_image_format &imageFormat, 29 const cl_image_desc &imageDesc, 30 bool zeroCopy, 31 MultiGraphicsAllocation multiGraphicsAllocation, 32 bool isImageRedescribed, 33 uint32_t baseMipLevel, 34 uint32_t mipCount, 35 const ClSurfaceFormatInfo *surfaceFormatInfo, 36 const SurfaceOffsets *surfaceOffsets); 37 38 struct ImageFactoryFuncs { 39 ImageCreatFunc createImageFunction; 40 }; 41 42 namespace ImageFunctions { 43 using ValidateAndCreateImageFunc = std::function<cl_mem(cl_context context, 44 const uint64_t *properties, 45 uint64_t flags, 46 uint64_t flagsIntel, 47 const cl_image_format *imageFormat, 48 const cl_image_desc *imageDesc, 49 const void *hostPtr, 50 int32_t &errcodeRet)>; 51 extern ValidateAndCreateImageFunc validateAndCreateImage; 52 } // namespace ImageFunctions 53 54 class Image : public MemObj { 55 public: 56 const static cl_ulong maskMagic = 0xFFFFFFFFFFFFFFFFLL; 57 static const cl_ulong objectMagic = MemObj::objectMagic | 0x01; 58 59 ~Image() override; 60 61 static Image *create(Context *context, 62 const MemoryProperties &memoryProperties, 63 cl_mem_flags flags, 64 cl_mem_flags_intel flagsIntel, 65 const ClSurfaceFormatInfo *surfaceFormat, 66 const cl_image_desc *imageDesc, 67 const void *hostPtr, 68 cl_int &errcodeRet); 69 70 static cl_mem validateAndCreateImage(cl_context context, 71 const cl_mem_properties *properties, 72 cl_mem_flags flags, 73 cl_mem_flags_intel flagsIntel, 74 const cl_image_format *imageFormat, 75 const cl_image_desc *imageDesc, 76 const void *hostPtr, 77 cl_int &errcodeRet); 78 79 static Image *createImageHw(Context *context, const MemoryProperties &memoryProperties, cl_mem_flags flags, 80 cl_mem_flags_intel flagsIntel, size_t size, void *hostPtr, 81 const cl_image_format &imageFormat, const cl_image_desc &imageDesc, 82 bool zeroCopy, MultiGraphicsAllocation multiGraphicsAllocation, 83 bool isObjectRedescribed, uint32_t baseMipLevel, uint32_t mipCount, const ClSurfaceFormatInfo *surfaceFormatInfo = nullptr); 84 85 static Image *createSharedImage(Context *context, SharingHandler *sharingHandler, const McsSurfaceInfo &mcsSurfaceInfo, 86 MultiGraphicsAllocation multiGraphicsAllocation, GraphicsAllocation *mcsAllocation, 87 cl_mem_flags flags, cl_mem_flags_intel flagsIntel, const ClSurfaceFormatInfo *surfaceFormat, ImageInfo &imgInfo, uint32_t cubeFaceIndex, uint32_t baseMipLevel, uint32_t mipCount); 88 89 static cl_int validate(Context *context, 90 const MemoryProperties &memoryProperties, 91 const ClSurfaceFormatInfo *surfaceFormat, 92 const cl_image_desc *imageDesc, 93 const void *hostPtr); 94 static cl_int validateImageFormat(const cl_image_format *imageFormat); 95 96 static int32_t validatePlanarYUV(Context *context, 97 const MemoryProperties &memoryProperties, 98 const cl_image_desc *imageDesc, 99 const void *hostPtr); 100 101 static int32_t validatePackedYUV(const MemoryProperties &memoryProperties, const cl_image_desc *imageDesc); 102 103 static cl_int validateImageTraits(Context *context, const MemoryProperties &memoryProperties, const cl_image_format *imageFormat, const cl_image_desc *imageDesc, const void *hostPtr); 104 105 static size_t calculateHostPtrSize(const size_t *region, size_t rowPitch, size_t slicePitch, size_t pixelSize, uint32_t imageType); 106 107 static void calculateHostPtrOffset(size_t *imageOffset, const size_t *origin, const size_t *region, size_t rowPitch, size_t slicePitch, uint32_t imageType, size_t bytesPerPixel); 108 109 static cl_int getImageParams(Context *context, 110 cl_mem_flags flags, 111 const ClSurfaceFormatInfo *surfaceFormat, 112 const cl_image_desc *imageDesc, 113 size_t *imageRowPitch, 114 size_t *imageSlicePitch); 115 116 static bool isImage1d(const cl_image_desc &imageDesc); 117 118 static bool isImage2d(cl_mem_object_type imageType); 119 120 static bool isImage2dOr2dArray(cl_mem_object_type imageType); 121 122 static bool isDepthFormat(const cl_image_format &imageFormat); 123 hasSlices(cl_mem_object_type type)124 static bool hasSlices(cl_mem_object_type type) { 125 return (type == CL_MEM_OBJECT_IMAGE3D) || (type == CL_MEM_OBJECT_IMAGE1D_ARRAY) || (type == CL_MEM_OBJECT_IMAGE2D_ARRAY); 126 } 127 128 static ImageType convertType(const cl_mem_object_type type); 129 static cl_mem_object_type convertType(const ImageType type); 130 static ImageDescriptor convertDescriptor(const cl_image_desc &imageDesc); 131 static cl_image_desc convertDescriptor(const ImageDescriptor &imageDesc); 132 133 cl_int getImageInfo(cl_image_info paramName, 134 size_t paramValueSize, 135 void *paramValue, 136 size_t *paramValueSizeRet); 137 138 virtual void setImageArg(void *memory, bool isMediaBlockImage, uint32_t mipLevel, uint32_t rootDeviceIndex, bool useGlobalAtomics) = 0; 139 virtual void setMediaImageArg(void *memory, uint32_t rootDeviceIndex) = 0; 140 virtual void setMediaSurfaceRotation(void *memory) = 0; 141 virtual void setSurfaceMemoryObjectControlState(void *memory, uint32_t value) = 0; 142 143 const cl_image_desc &getImageDesc() const; 144 const cl_image_format &getImageFormat() const; 145 const ClSurfaceFormatInfo &getSurfaceFormatInfo() const; 146 147 void transferDataToHostPtr(MemObjSizeArray ©Size, MemObjOffsetArray ©Offset) override; 148 void transferDataFromHostPtr(MemObjSizeArray ©Size, MemObjOffsetArray ©Offset) override; 149 150 Image *redescribe(); 151 Image *redescribeFillImage(); 152 ImageCreatFunc createFunction; 153 getQPitch()154 uint32_t getQPitch() { return qPitch; } setQPitch(uint32_t qPitch)155 void setQPitch(uint32_t qPitch) { this->qPitch = qPitch; } getHostPtrRowPitch()156 size_t getHostPtrRowPitch() const { return hostPtrRowPitch; } setHostPtrRowPitch(size_t pitch)157 void setHostPtrRowPitch(size_t pitch) { this->hostPtrRowPitch = pitch; } getHostPtrSlicePitch()158 size_t getHostPtrSlicePitch() const { return hostPtrSlicePitch; } setHostPtrSlicePitch(size_t pitch)159 void setHostPtrSlicePitch(size_t pitch) { this->hostPtrSlicePitch = pitch; } getImageCount()160 size_t getImageCount() const { return imageCount; } setImageCount(size_t imageCount)161 void setImageCount(size_t imageCount) { this->imageCount = imageCount; } setImageRowPitch(size_t rowPitch)162 void setImageRowPitch(size_t rowPitch) { imageDesc.image_row_pitch = rowPitch; } setImageSlicePitch(size_t slicePitch)163 void setImageSlicePitch(size_t slicePitch) { imageDesc.image_slice_pitch = slicePitch; } setSurfaceOffsets(uint64_t offset,uint32_t xOffset,uint32_t yOffset,uint32_t yOffsetForUVPlane)164 void setSurfaceOffsets(uint64_t offset, uint32_t xOffset, uint32_t yOffset, uint32_t yOffsetForUVPlane) { 165 surfaceOffsets.offset = offset; 166 surfaceOffsets.xOffset = xOffset; 167 surfaceOffsets.yOffset = yOffset; 168 surfaceOffsets.yOffsetForUVplane = yOffsetForUVPlane; 169 } getSurfaceOffsets(SurfaceOffsets & surfaceOffsetsOut)170 void getSurfaceOffsets(SurfaceOffsets &surfaceOffsetsOut) { surfaceOffsetsOut = this->surfaceOffsets; } 171 setCubeFaceIndex(uint32_t index)172 void setCubeFaceIndex(uint32_t index) { cubeFaceIndex = index; } getCubeFaceIndex()173 uint32_t getCubeFaceIndex() { return cubeFaceIndex; } setMediaPlaneType(cl_uint type)174 void setMediaPlaneType(cl_uint type) { mediaPlaneType = type; } getMediaPlaneType()175 cl_uint getMediaPlaneType() const { return mediaPlaneType; } peekBaseMipLevel()176 int peekBaseMipLevel() { return baseMipLevel; } setBaseMipLevel(int level)177 void setBaseMipLevel(int level) { this->baseMipLevel = level; } 178 peekMipCount()179 uint32_t peekMipCount() { return mipCount; } setMipCount(uint32_t mipCountNew)180 void setMipCount(uint32_t mipCountNew) { this->mipCount = mipCountNew; } 181 182 static const ClSurfaceFormatInfo *getSurfaceFormatFromTable(cl_mem_flags flags, const cl_image_format *imageFormat, bool supportsOcl20Features); 183 static cl_int validateRegionAndOrigin(const size_t *origin, const size_t *region, const cl_image_desc &imgDesc); 184 185 cl_int writeNV12Planes(const void *hostPtr, size_t hostPtrRowPitch, uint32_t rootDeviceIndex); setMcsSurfaceInfo(const McsSurfaceInfo & info)186 void setMcsSurfaceInfo(const McsSurfaceInfo &info) { mcsSurfaceInfo = info; } getMcsSurfaceInfo()187 const McsSurfaceInfo &getMcsSurfaceInfo() { return mcsSurfaceInfo; } 188 size_t calculateOffsetForMapping(const MemObjOffsetArray &origin) const override; 189 190 virtual void transformImage2dArrayTo3d(void *memory) = 0; 191 virtual void transformImage3dTo2dArray(void *memory) = 0; 192 193 bool hasSameDescriptor(const cl_image_desc &imageDesc) const; 194 bool hasValidParentImageFormat(const cl_image_format &imageFormat) const; 195 isImageFromBuffer()196 bool isImageFromBuffer() const { return castToObject<Buffer>(static_cast<cl_mem>(associatedMemObject)) ? true : false; } isImageFromImage()197 bool isImageFromImage() const { return castToObject<Image>(static_cast<cl_mem>(associatedMemObject)) ? true : false; } 198 199 static cl_int checkIfDeviceSupportsImages(cl_context context); 200 201 void fillImageRegion(size_t *region) const; 202 203 protected: 204 Image(Context *context, 205 const MemoryProperties &memoryProperties, 206 cl_mem_flags flags, 207 cl_mem_flags_intel flagsIntel, 208 size_t size, 209 void *memoryStorage, 210 void *hostPtr, 211 cl_image_format imageFormat, 212 const cl_image_desc &imageDesc, 213 bool zeroCopy, 214 MultiGraphicsAllocation multiGraphicsAllocation, 215 bool isObjectRedescribed, 216 uint32_t baseMipLevel, 217 uint32_t mipCount, 218 const ClSurfaceFormatInfo &surfaceFormatInfo, 219 const SurfaceOffsets *surfaceOffsets = nullptr); 220 221 void getOsSpecificImageInfo(const cl_mem_info ¶mName, size_t *srcParamSize, void **srcParam); 222 223 MOCKABLE_VIRTUAL void transferData(void *dst, size_t dstRowPitch, size_t dstSlicePitch, 224 void *src, size_t srcRowPitch, size_t srcSlicePitch, 225 std::array<size_t, 3> copyRegion, std::array<size_t, 3> copyOrigin); 226 227 cl_image_format imageFormat; 228 cl_image_desc imageDesc; 229 ClSurfaceFormatInfo surfaceFormatInfo; 230 McsSurfaceInfo mcsSurfaceInfo = {}; 231 uint32_t qPitch = 0; 232 size_t hostPtrRowPitch = 0; 233 size_t hostPtrSlicePitch = 0; 234 size_t imageCount = 0; 235 uint32_t cubeFaceIndex; 236 cl_uint mediaPlaneType; 237 SurfaceOffsets surfaceOffsets = {0}; 238 uint32_t baseMipLevel = 0; 239 uint32_t mipCount = 1; 240 241 static bool isValidSingleChannelFormat(const cl_image_format *imageFormat); 242 static bool isValidIntensityFormat(const cl_image_format *imageFormat); 243 static bool isValidLuminanceFormat(const cl_image_format *imageFormat); 244 static bool isValidDepthFormat(const cl_image_format *imageFormat); 245 static bool isValidDoubleChannelFormat(const cl_image_format *imageFormat); 246 static bool isValidTripleChannelFormat(const cl_image_format *imageFormat); 247 static bool isValidRGBAFormat(const cl_image_format *imageFormat); 248 static bool isValidSRGBFormat(const cl_image_format *imageFormat); 249 static bool isValidARGBFormat(const cl_image_format *imageFormat); 250 static bool isValidDepthStencilFormat(const cl_image_format *imageFormat); 251 static bool isValidYUVFormat(const cl_image_format *imageFormat); 252 static bool hasAlphaChannel(const cl_image_format *imageFormat); 253 }; 254 255 template <typename GfxFamily> 256 class ImageHw : public Image { 257 using RENDER_SURFACE_STATE = typename GfxFamily::RENDER_SURFACE_STATE; 258 using AUXILIARY_SURFACE_MODE = typename RENDER_SURFACE_STATE::AUXILIARY_SURFACE_MODE; 259 260 public: 261 ImageHw(Context *context, 262 const MemoryProperties &memoryProperties, 263 cl_mem_flags flags, 264 cl_mem_flags_intel flagsIntel, 265 size_t size, 266 void *memoryStorage, 267 void *hostPtr, 268 const cl_image_format &imageFormat, 269 const cl_image_desc &imageDesc, 270 bool zeroCopy, 271 MultiGraphicsAllocation multiGraphicsAllocation, 272 bool isObjectRedescribed, 273 uint32_t baseMipLevel, 274 uint32_t mipCount, 275 const ClSurfaceFormatInfo &surfaceFormatInfo, 276 const SurfaceOffsets *surfaceOffsets = nullptr) Image(context,memoryProperties,flags,flagsIntel,size,memoryStorage,hostPtr,imageFormat,imageDesc,zeroCopy,std::move (multiGraphicsAllocation),isObjectRedescribed,baseMipLevel,mipCount,surfaceFormatInfo,surfaceOffsets)277 : Image(context, memoryProperties, flags, flagsIntel, size, memoryStorage, hostPtr, imageFormat, imageDesc, 278 zeroCopy, std::move(multiGraphicsAllocation), isObjectRedescribed, baseMipLevel, mipCount, surfaceFormatInfo, surfaceOffsets) { 279 if (getImageDesc().image_type == CL_MEM_OBJECT_IMAGE1D || 280 getImageDesc().image_type == CL_MEM_OBJECT_IMAGE1D_BUFFER || 281 getImageDesc().image_type == CL_MEM_OBJECT_IMAGE2D || 282 getImageDesc().image_type == CL_MEM_OBJECT_IMAGE1D_ARRAY || 283 getImageDesc().image_type == CL_MEM_OBJECT_IMAGE2D_ARRAY) { 284 this->imageDesc.image_depth = 0; 285 } 286 287 switch (imageDesc.image_type) { 288 case CL_MEM_OBJECT_IMAGE1D: 289 case CL_MEM_OBJECT_IMAGE1D_BUFFER: 290 case CL_MEM_OBJECT_IMAGE1D_ARRAY: 291 surfaceType = RENDER_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_1D; 292 break; 293 default: 294 case CL_MEM_OBJECT_IMAGE2D_ARRAY: 295 case CL_MEM_OBJECT_IMAGE2D: 296 surfaceType = RENDER_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_2D; 297 break; 298 case CL_MEM_OBJECT_IMAGE3D: 299 surfaceType = RENDER_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_3D; 300 break; 301 } 302 } 303 304 void setImageArg(void *memory, bool setAsMediaBlockImage, uint32_t mipLevel, uint32_t rootDeviceIndex, bool useGlobalAtomics) override; 305 void setAuxParamsForMultisamples(RENDER_SURFACE_STATE *surfaceState); 306 void setMediaImageArg(void *memory, uint32_t rootDeviceIndex) override; 307 void setMediaSurfaceRotation(void *memory) override; 308 void setSurfaceMemoryObjectControlState(void *memory, uint32_t value) override; 309 void appendSurfaceStateParams(RENDER_SURFACE_STATE *surfaceState, uint32_t rootDeviceIndex, bool useGlobalAtomics); 310 void appendSurfaceStateDepthParams(RENDER_SURFACE_STATE *surfaceState, Gmm *gmm); 311 void appendSurfaceStateExt(void *memory); 312 void transformImage2dArrayTo3d(void *memory) override; 313 void transformImage3dTo2dArray(void *memory) override; create(Context * context,const MemoryProperties & memoryProperties,cl_mem_flags flags,cl_mem_flags_intel flagsIntel,size_t size,void * hostPtr,const cl_image_format & imageFormat,const cl_image_desc & imageDesc,bool zeroCopy,MultiGraphicsAllocation multiGraphicsAllocation,bool isObjectRedescribed,uint32_t baseMipLevel,uint32_t mipCount,const ClSurfaceFormatInfo * surfaceFormatInfo,const SurfaceOffsets * surfaceOffsets)314 static Image *create(Context *context, 315 const MemoryProperties &memoryProperties, 316 cl_mem_flags flags, 317 cl_mem_flags_intel flagsIntel, 318 size_t size, 319 void *hostPtr, 320 const cl_image_format &imageFormat, 321 const cl_image_desc &imageDesc, 322 bool zeroCopy, 323 MultiGraphicsAllocation multiGraphicsAllocation, 324 bool isObjectRedescribed, 325 uint32_t baseMipLevel, 326 uint32_t mipCount, 327 const ClSurfaceFormatInfo *surfaceFormatInfo, 328 const SurfaceOffsets *surfaceOffsets) { 329 UNRECOVERABLE_IF(surfaceFormatInfo == nullptr); 330 auto memoryStorage = multiGraphicsAllocation.getDefaultGraphicsAllocation()->getUnderlyingBuffer(); 331 return new ImageHw<GfxFamily>(context, 332 memoryProperties, 333 flags, 334 flagsIntel, 335 size, 336 memoryStorage, 337 hostPtr, 338 imageFormat, 339 imageDesc, 340 zeroCopy, 341 std::move(multiGraphicsAllocation), 342 isObjectRedescribed, 343 baseMipLevel, 344 mipCount, 345 *surfaceFormatInfo, 346 surfaceOffsets); 347 } 348 getShaderChannelValue(int inputShaderChannel,cl_channel_order imageChannelOrder)349 static int getShaderChannelValue(int inputShaderChannel, cl_channel_order imageChannelOrder) { 350 if (imageChannelOrder == CL_A) { 351 if (inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_RED || 352 inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_GREEN || 353 inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_BLUE) { 354 return RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_ZERO; 355 } 356 } else if (imageChannelOrder == CL_R || 357 imageChannelOrder == CL_RA || 358 imageChannelOrder == CL_Rx) { 359 if (inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_GREEN || 360 inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_BLUE) { 361 return RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_ZERO; 362 } 363 } else if (imageChannelOrder == CL_RG || 364 imageChannelOrder == CL_RGx) { 365 if (inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_BLUE) { 366 return RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_ZERO; 367 } 368 } 369 return inputShaderChannel; 370 } 371 typename RENDER_SURFACE_STATE::SURFACE_TYPE surfaceType; 372 }; 373 374 } // namespace NEO 375