1 /*
2  * Copyright (C) 2018-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #pragma once
9 #include "shared/source/helpers/string.h"
10 #include "shared/source/image/image_surface_state.h"
11 
12 #include "opencl/source/helpers/cl_validators.h"
13 #include "opencl/source/helpers/surface_formats.h"
14 #include "opencl/source/mem_obj/buffer.h"
15 #include "opencl/source/mem_obj/mem_obj.h"
16 
17 namespace NEO {
18 class Image;
19 struct KernelInfo;
20 struct SurfaceFormatInfo;
21 
22 using ImageCreatFunc = Image *(*)(Context *context,
23                                   const MemoryProperties &memoryProperties,
24                                   uint64_t flags,
25                                   uint64_t flagsIntel,
26                                   size_t size,
27                                   void *hostPtr,
28                                   const cl_image_format &imageFormat,
29                                   const cl_image_desc &imageDesc,
30                                   bool zeroCopy,
31                                   MultiGraphicsAllocation multiGraphicsAllocation,
32                                   bool isImageRedescribed,
33                                   uint32_t baseMipLevel,
34                                   uint32_t mipCount,
35                                   const ClSurfaceFormatInfo *surfaceFormatInfo,
36                                   const SurfaceOffsets *surfaceOffsets);
37 
38 struct ImageFactoryFuncs {
39     ImageCreatFunc createImageFunction;
40 };
41 
42 namespace ImageFunctions {
43 using ValidateAndCreateImageFunc = std::function<cl_mem(cl_context context,
44                                                         const uint64_t *properties,
45                                                         uint64_t flags,
46                                                         uint64_t flagsIntel,
47                                                         const cl_image_format *imageFormat,
48                                                         const cl_image_desc *imageDesc,
49                                                         const void *hostPtr,
50                                                         int32_t &errcodeRet)>;
51 extern ValidateAndCreateImageFunc validateAndCreateImage;
52 } // namespace ImageFunctions
53 
54 class Image : public MemObj {
55   public:
56     const static cl_ulong maskMagic = 0xFFFFFFFFFFFFFFFFLL;
57     static const cl_ulong objectMagic = MemObj::objectMagic | 0x01;
58 
59     ~Image() override;
60 
61     static Image *create(Context *context,
62                          const MemoryProperties &memoryProperties,
63                          cl_mem_flags flags,
64                          cl_mem_flags_intel flagsIntel,
65                          const ClSurfaceFormatInfo *surfaceFormat,
66                          const cl_image_desc *imageDesc,
67                          const void *hostPtr,
68                          cl_int &errcodeRet);
69 
70     static cl_mem validateAndCreateImage(cl_context context,
71                                          const cl_mem_properties *properties,
72                                          cl_mem_flags flags,
73                                          cl_mem_flags_intel flagsIntel,
74                                          const cl_image_format *imageFormat,
75                                          const cl_image_desc *imageDesc,
76                                          const void *hostPtr,
77                                          cl_int &errcodeRet);
78 
79     static Image *createImageHw(Context *context, const MemoryProperties &memoryProperties, cl_mem_flags flags,
80                                 cl_mem_flags_intel flagsIntel, size_t size, void *hostPtr,
81                                 const cl_image_format &imageFormat, const cl_image_desc &imageDesc,
82                                 bool zeroCopy, MultiGraphicsAllocation multiGraphicsAllocation,
83                                 bool isObjectRedescribed, uint32_t baseMipLevel, uint32_t mipCount, const ClSurfaceFormatInfo *surfaceFormatInfo = nullptr);
84 
85     static Image *createSharedImage(Context *context, SharingHandler *sharingHandler, const McsSurfaceInfo &mcsSurfaceInfo,
86                                     MultiGraphicsAllocation multiGraphicsAllocation, GraphicsAllocation *mcsAllocation,
87                                     cl_mem_flags flags, cl_mem_flags_intel flagsIntel, const ClSurfaceFormatInfo *surfaceFormat, ImageInfo &imgInfo, uint32_t cubeFaceIndex, uint32_t baseMipLevel, uint32_t mipCount);
88 
89     static cl_int validate(Context *context,
90                            const MemoryProperties &memoryProperties,
91                            const ClSurfaceFormatInfo *surfaceFormat,
92                            const cl_image_desc *imageDesc,
93                            const void *hostPtr);
94     static cl_int validateImageFormat(const cl_image_format *imageFormat);
95 
96     static int32_t validatePlanarYUV(Context *context,
97                                      const MemoryProperties &memoryProperties,
98                                      const cl_image_desc *imageDesc,
99                                      const void *hostPtr);
100 
101     static int32_t validatePackedYUV(const MemoryProperties &memoryProperties, const cl_image_desc *imageDesc);
102 
103     static cl_int validateImageTraits(Context *context, const MemoryProperties &memoryProperties, const cl_image_format *imageFormat, const cl_image_desc *imageDesc, const void *hostPtr);
104 
105     static size_t calculateHostPtrSize(const size_t *region, size_t rowPitch, size_t slicePitch, size_t pixelSize, uint32_t imageType);
106 
107     static void calculateHostPtrOffset(size_t *imageOffset, const size_t *origin, const size_t *region, size_t rowPitch, size_t slicePitch, uint32_t imageType, size_t bytesPerPixel);
108 
109     static cl_int getImageParams(Context *context,
110                                  cl_mem_flags flags,
111                                  const ClSurfaceFormatInfo *surfaceFormat,
112                                  const cl_image_desc *imageDesc,
113                                  size_t *imageRowPitch,
114                                  size_t *imageSlicePitch);
115 
116     static bool isImage1d(const cl_image_desc &imageDesc);
117 
118     static bool isImage2d(cl_mem_object_type imageType);
119 
120     static bool isImage2dOr2dArray(cl_mem_object_type imageType);
121 
122     static bool isDepthFormat(const cl_image_format &imageFormat);
123 
hasSlices(cl_mem_object_type type)124     static bool hasSlices(cl_mem_object_type type) {
125         return (type == CL_MEM_OBJECT_IMAGE3D) || (type == CL_MEM_OBJECT_IMAGE1D_ARRAY) || (type == CL_MEM_OBJECT_IMAGE2D_ARRAY);
126     }
127 
128     static ImageType convertType(const cl_mem_object_type type);
129     static cl_mem_object_type convertType(const ImageType type);
130     static ImageDescriptor convertDescriptor(const cl_image_desc &imageDesc);
131     static cl_image_desc convertDescriptor(const ImageDescriptor &imageDesc);
132 
133     cl_int getImageInfo(cl_image_info paramName,
134                         size_t paramValueSize,
135                         void *paramValue,
136                         size_t *paramValueSizeRet);
137 
138     virtual void setImageArg(void *memory, bool isMediaBlockImage, uint32_t mipLevel, uint32_t rootDeviceIndex, bool useGlobalAtomics) = 0;
139     virtual void setMediaImageArg(void *memory, uint32_t rootDeviceIndex) = 0;
140     virtual void setMediaSurfaceRotation(void *memory) = 0;
141     virtual void setSurfaceMemoryObjectControlState(void *memory, uint32_t value) = 0;
142 
143     const cl_image_desc &getImageDesc() const;
144     const cl_image_format &getImageFormat() const;
145     const ClSurfaceFormatInfo &getSurfaceFormatInfo() const;
146 
147     void transferDataToHostPtr(MemObjSizeArray &copySize, MemObjOffsetArray &copyOffset) override;
148     void transferDataFromHostPtr(MemObjSizeArray &copySize, MemObjOffsetArray &copyOffset) override;
149 
150     Image *redescribe();
151     Image *redescribeFillImage();
152     ImageCreatFunc createFunction;
153 
getQPitch()154     uint32_t getQPitch() { return qPitch; }
setQPitch(uint32_t qPitch)155     void setQPitch(uint32_t qPitch) { this->qPitch = qPitch; }
getHostPtrRowPitch()156     size_t getHostPtrRowPitch() const { return hostPtrRowPitch; }
setHostPtrRowPitch(size_t pitch)157     void setHostPtrRowPitch(size_t pitch) { this->hostPtrRowPitch = pitch; }
getHostPtrSlicePitch()158     size_t getHostPtrSlicePitch() const { return hostPtrSlicePitch; }
setHostPtrSlicePitch(size_t pitch)159     void setHostPtrSlicePitch(size_t pitch) { this->hostPtrSlicePitch = pitch; }
getImageCount()160     size_t getImageCount() const { return imageCount; }
setImageCount(size_t imageCount)161     void setImageCount(size_t imageCount) { this->imageCount = imageCount; }
setImageRowPitch(size_t rowPitch)162     void setImageRowPitch(size_t rowPitch) { imageDesc.image_row_pitch = rowPitch; }
setImageSlicePitch(size_t slicePitch)163     void setImageSlicePitch(size_t slicePitch) { imageDesc.image_slice_pitch = slicePitch; }
setSurfaceOffsets(uint64_t offset,uint32_t xOffset,uint32_t yOffset,uint32_t yOffsetForUVPlane)164     void setSurfaceOffsets(uint64_t offset, uint32_t xOffset, uint32_t yOffset, uint32_t yOffsetForUVPlane) {
165         surfaceOffsets.offset = offset;
166         surfaceOffsets.xOffset = xOffset;
167         surfaceOffsets.yOffset = yOffset;
168         surfaceOffsets.yOffsetForUVplane = yOffsetForUVPlane;
169     }
getSurfaceOffsets(SurfaceOffsets & surfaceOffsetsOut)170     void getSurfaceOffsets(SurfaceOffsets &surfaceOffsetsOut) { surfaceOffsetsOut = this->surfaceOffsets; }
171 
setCubeFaceIndex(uint32_t index)172     void setCubeFaceIndex(uint32_t index) { cubeFaceIndex = index; }
getCubeFaceIndex()173     uint32_t getCubeFaceIndex() { return cubeFaceIndex; }
setMediaPlaneType(cl_uint type)174     void setMediaPlaneType(cl_uint type) { mediaPlaneType = type; }
getMediaPlaneType()175     cl_uint getMediaPlaneType() const { return mediaPlaneType; }
peekBaseMipLevel()176     int peekBaseMipLevel() { return baseMipLevel; }
setBaseMipLevel(int level)177     void setBaseMipLevel(int level) { this->baseMipLevel = level; }
178 
peekMipCount()179     uint32_t peekMipCount() { return mipCount; }
setMipCount(uint32_t mipCountNew)180     void setMipCount(uint32_t mipCountNew) { this->mipCount = mipCountNew; }
181 
182     static const ClSurfaceFormatInfo *getSurfaceFormatFromTable(cl_mem_flags flags, const cl_image_format *imageFormat, bool supportsOcl20Features);
183     static cl_int validateRegionAndOrigin(const size_t *origin, const size_t *region, const cl_image_desc &imgDesc);
184 
185     cl_int writeNV12Planes(const void *hostPtr, size_t hostPtrRowPitch, uint32_t rootDeviceIndex);
setMcsSurfaceInfo(const McsSurfaceInfo & info)186     void setMcsSurfaceInfo(const McsSurfaceInfo &info) { mcsSurfaceInfo = info; }
getMcsSurfaceInfo()187     const McsSurfaceInfo &getMcsSurfaceInfo() { return mcsSurfaceInfo; }
188     size_t calculateOffsetForMapping(const MemObjOffsetArray &origin) const override;
189 
190     virtual void transformImage2dArrayTo3d(void *memory) = 0;
191     virtual void transformImage3dTo2dArray(void *memory) = 0;
192 
193     bool hasSameDescriptor(const cl_image_desc &imageDesc) const;
194     bool hasValidParentImageFormat(const cl_image_format &imageFormat) const;
195 
isImageFromBuffer()196     bool isImageFromBuffer() const { return castToObject<Buffer>(static_cast<cl_mem>(associatedMemObject)) ? true : false; }
isImageFromImage()197     bool isImageFromImage() const { return castToObject<Image>(static_cast<cl_mem>(associatedMemObject)) ? true : false; }
198 
199     static cl_int checkIfDeviceSupportsImages(cl_context context);
200 
201     void fillImageRegion(size_t *region) const;
202 
203   protected:
204     Image(Context *context,
205           const MemoryProperties &memoryProperties,
206           cl_mem_flags flags,
207           cl_mem_flags_intel flagsIntel,
208           size_t size,
209           void *memoryStorage,
210           void *hostPtr,
211           cl_image_format imageFormat,
212           const cl_image_desc &imageDesc,
213           bool zeroCopy,
214           MultiGraphicsAllocation multiGraphicsAllocation,
215           bool isObjectRedescribed,
216           uint32_t baseMipLevel,
217           uint32_t mipCount,
218           const ClSurfaceFormatInfo &surfaceFormatInfo,
219           const SurfaceOffsets *surfaceOffsets = nullptr);
220 
221     void getOsSpecificImageInfo(const cl_mem_info &paramName, size_t *srcParamSize, void **srcParam);
222 
223     MOCKABLE_VIRTUAL void transferData(void *dst, size_t dstRowPitch, size_t dstSlicePitch,
224                                        void *src, size_t srcRowPitch, size_t srcSlicePitch,
225                                        std::array<size_t, 3> copyRegion, std::array<size_t, 3> copyOrigin);
226 
227     cl_image_format imageFormat;
228     cl_image_desc imageDesc;
229     ClSurfaceFormatInfo surfaceFormatInfo;
230     McsSurfaceInfo mcsSurfaceInfo = {};
231     uint32_t qPitch = 0;
232     size_t hostPtrRowPitch = 0;
233     size_t hostPtrSlicePitch = 0;
234     size_t imageCount = 0;
235     uint32_t cubeFaceIndex;
236     cl_uint mediaPlaneType;
237     SurfaceOffsets surfaceOffsets = {0};
238     uint32_t baseMipLevel = 0;
239     uint32_t mipCount = 1;
240 
241     static bool isValidSingleChannelFormat(const cl_image_format *imageFormat);
242     static bool isValidIntensityFormat(const cl_image_format *imageFormat);
243     static bool isValidLuminanceFormat(const cl_image_format *imageFormat);
244     static bool isValidDepthFormat(const cl_image_format *imageFormat);
245     static bool isValidDoubleChannelFormat(const cl_image_format *imageFormat);
246     static bool isValidTripleChannelFormat(const cl_image_format *imageFormat);
247     static bool isValidRGBAFormat(const cl_image_format *imageFormat);
248     static bool isValidSRGBFormat(const cl_image_format *imageFormat);
249     static bool isValidARGBFormat(const cl_image_format *imageFormat);
250     static bool isValidDepthStencilFormat(const cl_image_format *imageFormat);
251     static bool isValidYUVFormat(const cl_image_format *imageFormat);
252     static bool hasAlphaChannel(const cl_image_format *imageFormat);
253 };
254 
255 template <typename GfxFamily>
256 class ImageHw : public Image {
257     using RENDER_SURFACE_STATE = typename GfxFamily::RENDER_SURFACE_STATE;
258     using AUXILIARY_SURFACE_MODE = typename RENDER_SURFACE_STATE::AUXILIARY_SURFACE_MODE;
259 
260   public:
261     ImageHw(Context *context,
262             const MemoryProperties &memoryProperties,
263             cl_mem_flags flags,
264             cl_mem_flags_intel flagsIntel,
265             size_t size,
266             void *memoryStorage,
267             void *hostPtr,
268             const cl_image_format &imageFormat,
269             const cl_image_desc &imageDesc,
270             bool zeroCopy,
271             MultiGraphicsAllocation multiGraphicsAllocation,
272             bool isObjectRedescribed,
273             uint32_t baseMipLevel,
274             uint32_t mipCount,
275             const ClSurfaceFormatInfo &surfaceFormatInfo,
276             const SurfaceOffsets *surfaceOffsets = nullptr)
Image(context,memoryProperties,flags,flagsIntel,size,memoryStorage,hostPtr,imageFormat,imageDesc,zeroCopy,std::move (multiGraphicsAllocation),isObjectRedescribed,baseMipLevel,mipCount,surfaceFormatInfo,surfaceOffsets)277         : Image(context, memoryProperties, flags, flagsIntel, size, memoryStorage, hostPtr, imageFormat, imageDesc,
278                 zeroCopy, std::move(multiGraphicsAllocation), isObjectRedescribed, baseMipLevel, mipCount, surfaceFormatInfo, surfaceOffsets) {
279         if (getImageDesc().image_type == CL_MEM_OBJECT_IMAGE1D ||
280             getImageDesc().image_type == CL_MEM_OBJECT_IMAGE1D_BUFFER ||
281             getImageDesc().image_type == CL_MEM_OBJECT_IMAGE2D ||
282             getImageDesc().image_type == CL_MEM_OBJECT_IMAGE1D_ARRAY ||
283             getImageDesc().image_type == CL_MEM_OBJECT_IMAGE2D_ARRAY) {
284             this->imageDesc.image_depth = 0;
285         }
286 
287         switch (imageDesc.image_type) {
288         case CL_MEM_OBJECT_IMAGE1D:
289         case CL_MEM_OBJECT_IMAGE1D_BUFFER:
290         case CL_MEM_OBJECT_IMAGE1D_ARRAY:
291             surfaceType = RENDER_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_1D;
292             break;
293         default:
294         case CL_MEM_OBJECT_IMAGE2D_ARRAY:
295         case CL_MEM_OBJECT_IMAGE2D:
296             surfaceType = RENDER_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_2D;
297             break;
298         case CL_MEM_OBJECT_IMAGE3D:
299             surfaceType = RENDER_SURFACE_STATE::SURFACE_TYPE_SURFTYPE_3D;
300             break;
301         }
302     }
303 
304     void setImageArg(void *memory, bool setAsMediaBlockImage, uint32_t mipLevel, uint32_t rootDeviceIndex, bool useGlobalAtomics) override;
305     void setAuxParamsForMultisamples(RENDER_SURFACE_STATE *surfaceState);
306     void setMediaImageArg(void *memory, uint32_t rootDeviceIndex) override;
307     void setMediaSurfaceRotation(void *memory) override;
308     void setSurfaceMemoryObjectControlState(void *memory, uint32_t value) override;
309     void appendSurfaceStateParams(RENDER_SURFACE_STATE *surfaceState, uint32_t rootDeviceIndex, bool useGlobalAtomics);
310     void appendSurfaceStateDepthParams(RENDER_SURFACE_STATE *surfaceState, Gmm *gmm);
311     void appendSurfaceStateExt(void *memory);
312     void transformImage2dArrayTo3d(void *memory) override;
313     void transformImage3dTo2dArray(void *memory) override;
create(Context * context,const MemoryProperties & memoryProperties,cl_mem_flags flags,cl_mem_flags_intel flagsIntel,size_t size,void * hostPtr,const cl_image_format & imageFormat,const cl_image_desc & imageDesc,bool zeroCopy,MultiGraphicsAllocation multiGraphicsAllocation,bool isObjectRedescribed,uint32_t baseMipLevel,uint32_t mipCount,const ClSurfaceFormatInfo * surfaceFormatInfo,const SurfaceOffsets * surfaceOffsets)314     static Image *create(Context *context,
315                          const MemoryProperties &memoryProperties,
316                          cl_mem_flags flags,
317                          cl_mem_flags_intel flagsIntel,
318                          size_t size,
319                          void *hostPtr,
320                          const cl_image_format &imageFormat,
321                          const cl_image_desc &imageDesc,
322                          bool zeroCopy,
323                          MultiGraphicsAllocation multiGraphicsAllocation,
324                          bool isObjectRedescribed,
325                          uint32_t baseMipLevel,
326                          uint32_t mipCount,
327                          const ClSurfaceFormatInfo *surfaceFormatInfo,
328                          const SurfaceOffsets *surfaceOffsets) {
329         UNRECOVERABLE_IF(surfaceFormatInfo == nullptr);
330         auto memoryStorage = multiGraphicsAllocation.getDefaultGraphicsAllocation()->getUnderlyingBuffer();
331         return new ImageHw<GfxFamily>(context,
332                                       memoryProperties,
333                                       flags,
334                                       flagsIntel,
335                                       size,
336                                       memoryStorage,
337                                       hostPtr,
338                                       imageFormat,
339                                       imageDesc,
340                                       zeroCopy,
341                                       std::move(multiGraphicsAllocation),
342                                       isObjectRedescribed,
343                                       baseMipLevel,
344                                       mipCount,
345                                       *surfaceFormatInfo,
346                                       surfaceOffsets);
347     }
348 
getShaderChannelValue(int inputShaderChannel,cl_channel_order imageChannelOrder)349     static int getShaderChannelValue(int inputShaderChannel, cl_channel_order imageChannelOrder) {
350         if (imageChannelOrder == CL_A) {
351             if (inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_RED ||
352                 inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_GREEN ||
353                 inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_BLUE) {
354                 return RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_ZERO;
355             }
356         } else if (imageChannelOrder == CL_R ||
357                    imageChannelOrder == CL_RA ||
358                    imageChannelOrder == CL_Rx) {
359             if (inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_GREEN ||
360                 inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_BLUE) {
361                 return RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_ZERO;
362             }
363         } else if (imageChannelOrder == CL_RG ||
364                    imageChannelOrder == CL_RGx) {
365             if (inputShaderChannel == RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_BLUE) {
366                 return RENDER_SURFACE_STATE::SHADER_CHANNEL_SELECT_ZERO;
367             }
368         }
369         return inputShaderChannel;
370     }
371     typename RENDER_SURFACE_STATE::SURFACE_TYPE surfaceType;
372 };
373 
374 } // namespace NEO
375