1 /*
2  * Copyright (C) 2018-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #include "shared/source/gmm_helper/gmm.h"
9 
10 #include "shared/source/gmm_helper/client_context/gmm_client_context.h"
11 #include "shared/source/gmm_helper/gmm_helper.h"
12 #include "shared/source/gmm_helper/resource_info.h"
13 #include "shared/source/helpers/aligned_memory.h"
14 #include "shared/source/helpers/debug_helpers.h"
15 #include "shared/source/helpers/hw_helper.h"
16 #include "shared/source/helpers/hw_info.h"
17 #include "shared/source/helpers/ptr_math.h"
18 #include "shared/source/helpers/surface_format_info.h"
19 
20 namespace NEO {
Gmm(GmmClientContext * clientContext,const void * alignedPtr,size_t alignedSize,size_t alignment,bool uncacheable)21 Gmm::Gmm(GmmClientContext *clientContext, const void *alignedPtr, size_t alignedSize, size_t alignment, bool uncacheable) : Gmm(clientContext, alignedPtr, alignedSize, alignment, uncacheable, false, true, {}) {}
22 
Gmm(GmmClientContext * clientContext,const void * alignedPtr,size_t alignedSize,size_t alignment,bool uncacheable,bool preferCompressed,bool systemMemoryPool,StorageInfo storageInfo)23 Gmm::Gmm(GmmClientContext *clientContext, const void *alignedPtr, size_t alignedSize, size_t alignment, bool uncacheable, bool preferCompressed, bool systemMemoryPool, StorageInfo storageInfo)
24     : Gmm(clientContext, alignedPtr, alignedSize, alignment, uncacheable, preferCompressed, systemMemoryPool, storageInfo, true) {
25 }
26 
Gmm(GmmClientContext * clientContext,const void * alignedPtr,size_t alignedSize,size_t alignment,bool uncacheable,bool preferCompressed,bool systemMemoryPool,StorageInfo storageInfo,bool allowLargePages)27 Gmm::Gmm(GmmClientContext *clientContext, const void *alignedPtr, size_t alignedSize, size_t alignment, bool uncacheable, bool preferCompressed, bool systemMemoryPool, StorageInfo storageInfo, bool allowLargePages) : clientContext(clientContext) {
28     resourceParams.Type = RESOURCE_BUFFER;
29     resourceParams.Format = GMM_FORMAT_GENERIC_8BIT;
30     resourceParams.BaseWidth64 = static_cast<uint64_t>(alignedSize);
31     resourceParams.BaseHeight = 1;
32     resourceParams.Depth = 1;
33     resourceParams.BaseAlignment = static_cast<uint32_t>(alignment);
34     if ((nullptr == alignedPtr) && (false == allowLargePages)) {
35         resourceParams.Flags.Info.NoOptimizationPadding = true;
36         if ((resourceParams.BaseWidth64 & MemoryConstants::page64kMask) == 0) {
37             resourceParams.BaseWidth64 += MemoryConstants::pageSize;
38         }
39     }
40     if (!uncacheable) {
41         resourceParams.Usage = GMM_RESOURCE_USAGE_OCL_BUFFER;
42     } else {
43         resourceParams.Usage = GMM_RESOURCE_USAGE_OCL_BUFFER_CSR_UC;
44     }
45     resourceParams.Flags.Info.Linear = 1;
46     resourceParams.Flags.Info.Cacheable = 1;
47     resourceParams.Flags.Gpu.Texture = 1;
48 
49     if (alignedPtr) {
50         resourceParams.Flags.Info.ExistingSysMem = 1;
51         resourceParams.pExistingSysMem = castToUint64(alignedPtr);
52         resourceParams.ExistingSysMemSize = alignedSize;
53     } else {
54         resourceParams.NoGfxMemory = 1u;
55     }
56 
57     if (resourceParams.BaseWidth64 >= GmmHelper::maxPossiblePitch) {
58         resourceParams.Flags.Gpu.NoRestriction = 1;
59     }
60 
61     applyAuxFlagsForBuffer(preferCompressed);
62     applyMemoryFlags(systemMemoryPool, storageInfo);
63     applyAppResource(storageInfo);
64     applyDebugOverrides();
65 
66     gmmResourceInfo.reset(GmmResourceInfo::create(clientContext, &resourceParams));
67 }
68 
Gmm(GmmClientContext * clientContext,GMM_RESOURCE_INFO * inputGmm)69 Gmm::Gmm(GmmClientContext *clientContext, GMM_RESOURCE_INFO *inputGmm) : clientContext(clientContext) {
70     gmmResourceInfo.reset(GmmResourceInfo::create(clientContext, inputGmm));
71     applyDebugOverrides();
72 }
73 
74 Gmm::~Gmm() = default;
75 
Gmm(GmmClientContext * clientContext,ImageInfo & inputOutputImgInfo,StorageInfo storageInfo,bool preferCompressed)76 Gmm::Gmm(GmmClientContext *clientContext, ImageInfo &inputOutputImgInfo, StorageInfo storageInfo, bool preferCompressed) : clientContext(clientContext) {
77     this->resourceParams = {};
78     setupImageResourceParams(inputOutputImgInfo, preferCompressed);
79     applyMemoryFlags(!inputOutputImgInfo.useLocalMemory, storageInfo);
80     applyAppResource(storageInfo);
81     applyDebugOverrides();
82 
83     this->gmmResourceInfo.reset(GmmResourceInfo::create(clientContext, &this->resourceParams));
84     UNRECOVERABLE_IF(this->gmmResourceInfo == nullptr);
85 
86     queryImageParams(inputOutputImgInfo);
87 }
88 
setupImageResourceParams(ImageInfo & imgInfo,bool preferCompressed)89 void Gmm::setupImageResourceParams(ImageInfo &imgInfo, bool preferCompressed) {
90     uint64_t imageWidth = static_cast<uint64_t>(imgInfo.imgDesc.imageWidth);
91     uint32_t imageHeight = 1;
92     uint32_t imageDepth = 1;
93     uint32_t imageCount = 1;
94 
95     switch (imgInfo.imgDesc.imageType) {
96     case ImageType::Image1D:
97     case ImageType::Image1DArray:
98     case ImageType::Image1DBuffer:
99         resourceParams.Type = GMM_RESOURCE_TYPE::RESOURCE_1D;
100         break;
101     case ImageType::Image2D:
102     case ImageType::Image2DArray:
103         resourceParams.Type = GMM_RESOURCE_TYPE::RESOURCE_2D;
104         imageHeight = static_cast<uint32_t>(imgInfo.imgDesc.imageHeight);
105         break;
106     case ImageType::Image3D:
107         resourceParams.Type = GMM_RESOURCE_TYPE::RESOURCE_3D;
108         imageHeight = static_cast<uint32_t>(imgInfo.imgDesc.imageHeight);
109         imageDepth = static_cast<uint32_t>(imgInfo.imgDesc.imageDepth);
110         break;
111     default:
112         return;
113     }
114 
115     if (imgInfo.imgDesc.imageType == ImageType::Image1DArray ||
116         imgInfo.imgDesc.imageType == ImageType::Image2DArray) {
117         imageCount = static_cast<uint32_t>(imgInfo.imgDesc.imageArraySize);
118     }
119 
120     resourceParams.Flags.Info.Linear = imgInfo.linearStorage;
121 
122     auto &hwHelper = HwHelper::get(clientContext->getHardwareInfo()->platform.eRenderCoreFamily);
123 
124     resourceParams.NoGfxMemory = 1; // dont allocate, only query for params
125 
126     resourceParams.Usage = GMM_RESOURCE_USAGE_TYPE::GMM_RESOURCE_USAGE_OCL_IMAGE;
127 
128     resourceParams.Format = imgInfo.surfaceFormat->GMMSurfaceFormat;
129     resourceParams.Flags.Gpu.Texture = 1;
130     resourceParams.BaseWidth64 = imageWidth;
131     resourceParams.BaseHeight = imageHeight;
132     resourceParams.Depth = imageDepth;
133     resourceParams.ArraySize = imageCount;
134     resourceParams.Flags.Wa.__ForceOtherHVALIGN4 = hwHelper.hvAlign4Required();
135     resourceParams.MaxLod = imgInfo.baseMipLevel + imgInfo.mipCount;
136     if (imgInfo.imgDesc.imageRowPitch && imgInfo.imgDesc.fromParent) {
137         resourceParams.OverridePitch = (uint32_t)imgInfo.imgDesc.imageRowPitch;
138         resourceParams.Flags.Info.AllowVirtualPadding = true;
139     }
140 
141     applyAuxFlagsForImage(imgInfo, preferCompressed);
142 }
143 
applyAuxFlagsForBuffer(bool preferCompression)144 void Gmm::applyAuxFlagsForBuffer(bool preferCompression) {
145     auto hardwareInfo = clientContext->getHardwareInfo();
146     bool allowCompression = HwHelper::compressedBuffersSupported(*hardwareInfo) &&
147                             preferCompression;
148 
149     auto &hwHelper = HwHelper::get(hardwareInfo->platform.eRenderCoreFamily);
150     if (allowCompression) {
151         hwHelper.applyRenderCompressionFlag(*this, 1);
152         resourceParams.Flags.Gpu.CCS = 1;
153         resourceParams.Flags.Gpu.UnifiedAuxSurface = 1;
154         isCompressionEnabled = true;
155     }
156     hwHelper.applyAdditionalCompressionSettings(*this, !isCompressionEnabled);
157 }
158 
applyAuxFlagsForImage(ImageInfo & imgInfo,bool preferCompressed)159 void Gmm::applyAuxFlagsForImage(ImageInfo &imgInfo, bool preferCompressed) {
160     uint8_t compressionFormat;
161     if (this->resourceParams.Flags.Info.MediaCompressed) {
162         compressionFormat = clientContext->getMediaSurfaceStateCompressionFormat(imgInfo.surfaceFormat->GMMSurfaceFormat);
163     } else {
164         compressionFormat = clientContext->getSurfaceStateCompressionFormat(imgInfo.surfaceFormat->GMMSurfaceFormat);
165     }
166 
167     bool compressionFormatSupported = false;
168     if (clientContext->getHardwareInfo()->featureTable.flags.ftrFlatPhysCCS) {
169         compressionFormatSupported = compressionFormat != GMM_FLATCCS_FORMAT::GMM_FLATCCS_FORMAT_INVALID;
170     } else {
171         compressionFormatSupported = compressionFormat != GMM_E2ECOMP_FORMAT::GMM_E2ECOMP_FORMAT_INVALID;
172     }
173 
174     const bool isPackedYuv = imgInfo.surfaceFormat->GMMSurfaceFormat == GMM_FORMAT_YUY2 ||
175                              imgInfo.surfaceFormat->GMMSurfaceFormat == GMM_FORMAT_UYVY ||
176                              imgInfo.surfaceFormat->GMMSurfaceFormat == GMM_FORMAT_YVYU ||
177                              imgInfo.surfaceFormat->GMMSurfaceFormat == GMM_FORMAT_VYUY;
178 
179     auto hwInfo = clientContext->getHardwareInfo();
180 
181     bool allowCompression = HwHelper::compressedImagesSupported(*hwInfo) &&
182                             preferCompressed &&
183                             compressionFormatSupported &&
184                             imgInfo.surfaceFormat->GMMSurfaceFormat != GMM_RESOURCE_FORMAT::GMM_FORMAT_NV12 &&
185                             imgInfo.plane == GMM_YUV_PLANE_ENUM::GMM_NO_PLANE &&
186                             !isPackedYuv;
187 
188     auto &hwHelper = HwHelper::get(hwInfo->platform.eRenderCoreFamily);
189     if (imgInfo.useLocalMemory || !hwInfo->featureTable.flags.ftrLocalMemory) {
190         if (allowCompression) {
191             hwHelper.applyRenderCompressionFlag(*this, 1);
192             this->resourceParams.Flags.Gpu.CCS = 1;
193             this->resourceParams.Flags.Gpu.UnifiedAuxSurface = 1;
194             this->resourceParams.Flags.Gpu.IndirectClearColor = 1;
195             this->isCompressionEnabled = true;
196         }
197     }
198     hwHelper.applyAdditionalCompressionSettings(*this, !isCompressionEnabled);
199 }
200 
queryImageParams(ImageInfo & imgInfo)201 void Gmm::queryImageParams(ImageInfo &imgInfo) {
202     auto imageCount = this->gmmResourceInfo->getArraySize();
203     imgInfo.size = this->gmmResourceInfo->getSizeAllocation();
204 
205     imgInfo.rowPitch = this->gmmResourceInfo->getRenderPitch();
206     if (imgInfo.rowPitch == 0) { // WA
207         imgInfo.rowPitch = alignUp(this->gmmResourceInfo->getBaseWidth(), this->gmmResourceInfo->getHAlign());
208         imgInfo.rowPitch = imgInfo.rowPitch * (this->gmmResourceInfo->getBitsPerPixel() >> 3);
209     }
210 
211     // calculate slice pitch
212     if ((this->resourceParams.Type == GMM_RESOURCE_TYPE::RESOURCE_2D ||
213          this->resourceParams.Type == GMM_RESOURCE_TYPE::RESOURCE_1D) &&
214         imageCount == 1) {
215         // 2D or 1D or 1Darray with array_size=1
216         imgInfo.slicePitch = imgInfo.size;
217     } else {
218         // 3D Image or 2D-Array or 1D-Arrays (array_size>1)
219         GMM_REQ_OFFSET_INFO reqOffsetInfo = {};
220         reqOffsetInfo.ReqRender = 1;
221         reqOffsetInfo.Slice = 1;
222         reqOffsetInfo.ArrayIndex = (imageCount > 1) ? 1 : 0;
223 
224         this->gmmResourceInfo->getOffset(reqOffsetInfo);
225         imgInfo.slicePitch = reqOffsetInfo.Render.Offset;
226         imgInfo.slicePitch += imgInfo.rowPitch * reqOffsetInfo.Render.YOffset;
227         imgInfo.slicePitch += reqOffsetInfo.Render.XOffset;
228     }
229 
230     if (imgInfo.plane != GMM_NO_PLANE) {
231         GMM_REQ_OFFSET_INFO reqOffsetInfo = {};
232         reqOffsetInfo.ReqRender = 1;
233         reqOffsetInfo.Slice = 0;
234         reqOffsetInfo.ArrayIndex = 0;
235         reqOffsetInfo.Plane = imgInfo.plane;
236         this->gmmResourceInfo->getOffset(reqOffsetInfo);
237         imgInfo.xOffset = reqOffsetInfo.Render.XOffset / (this->gmmResourceInfo->getBitsPerPixel() / 8);
238         imgInfo.yOffset = reqOffsetInfo.Render.YOffset;
239         imgInfo.offset = reqOffsetInfo.Render.Offset;
240     }
241 
242     if (imgInfo.surfaceFormat->GMMSurfaceFormat == GMM_RESOURCE_FORMAT::GMM_FORMAT_NV12 || imgInfo.surfaceFormat->GMMSurfaceFormat == GMM_RESOURCE_FORMAT::GMM_FORMAT_P010) {
243         GMM_REQ_OFFSET_INFO reqOffsetInfo = {};
244         reqOffsetInfo.ReqLock = 1;
245         reqOffsetInfo.Slice = 1;
246         reqOffsetInfo.ArrayIndex = 0;
247         reqOffsetInfo.Plane = GMM_PLANE_U;
248         this->gmmResourceInfo->getOffset(reqOffsetInfo);
249         UNRECOVERABLE_IF(reqOffsetInfo.Lock.Pitch == 0);
250         imgInfo.yOffsetForUVPlane = reqOffsetInfo.Lock.Offset / reqOffsetInfo.Lock.Pitch;
251     }
252 
253     imgInfo.qPitch = queryQPitch(this->resourceParams.Type);
254 }
255 
queryQPitch(GMM_RESOURCE_TYPE resType)256 uint32_t Gmm::queryQPitch(GMM_RESOURCE_TYPE resType) {
257     if (clientContext->getHardwareInfo()->platform.eRenderCoreFamily == IGFX_GEN8_CORE && resType == GMM_RESOURCE_TYPE::RESOURCE_3D) {
258         return 0;
259     }
260     return gmmResourceInfo->getQPitch();
261 }
262 
updateImgInfoAndDesc(ImageInfo & imgInfo,uint32_t arrayIndex)263 void Gmm::updateImgInfoAndDesc(ImageInfo &imgInfo, uint32_t arrayIndex) {
264     imgInfo.imgDesc.imageWidth = gmmResourceInfo->getBaseWidth();
265     imgInfo.imgDesc.imageRowPitch = gmmResourceInfo->getRenderPitch();
266     if (imgInfo.imgDesc.imageRowPitch == 0) {
267         size_t width = alignUp(imgInfo.imgDesc.imageWidth, gmmResourceInfo->getHAlign());
268         imgInfo.imgDesc.imageRowPitch = width * (gmmResourceInfo->getBitsPerPixel() >> 3);
269     }
270     imgInfo.imgDesc.imageHeight = gmmResourceInfo->getBaseHeight();
271     imgInfo.imgDesc.imageDepth = gmmResourceInfo->getBaseDepth();
272     imgInfo.imgDesc.imageArraySize = gmmResourceInfo->getArraySize();
273     if (imgInfo.imgDesc.imageDepth > 1 || imgInfo.imgDesc.imageArraySize > 1) {
274         GMM_REQ_OFFSET_INFO reqOffsetInfo = {};
275         reqOffsetInfo.Slice = imgInfo.imgDesc.imageDepth > 1 ? 1 : 0;
276         reqOffsetInfo.ArrayIndex = imgInfo.imgDesc.imageArraySize > 1 ? 1 : 0;
277         reqOffsetInfo.ReqLock = 1;
278         gmmResourceInfo->getOffset(reqOffsetInfo);
279         imgInfo.imgDesc.imageSlicePitch = static_cast<size_t>(reqOffsetInfo.Lock.Offset);
280     } else {
281         imgInfo.imgDesc.imageSlicePitch = gmmResourceInfo->getSizeAllocation();
282     }
283 
284     updateOffsetsInImgInfo(imgInfo, arrayIndex);
285 }
286 
updateOffsetsInImgInfo(ImageInfo & imgInfo,uint32_t arrayIndex)287 void Gmm::updateOffsetsInImgInfo(ImageInfo &imgInfo, uint32_t arrayIndex) {
288     GMM_REQ_OFFSET_INFO reqOffsetInfo = {};
289     reqOffsetInfo.ReqRender = 1;
290     reqOffsetInfo.Slice = 0;
291     reqOffsetInfo.ArrayIndex = arrayIndex;
292     reqOffsetInfo.Plane = imgInfo.plane;
293     gmmResourceInfo->getOffset(reqOffsetInfo);
294     UNRECOVERABLE_IF(gmmResourceInfo->getBitsPerPixel() == 0u);
295     imgInfo.xOffset = reqOffsetInfo.Render.XOffset / (gmmResourceInfo->getBitsPerPixel() / 8);
296     imgInfo.yOffset = reqOffsetInfo.Render.YOffset;
297     imgInfo.offset = reqOffsetInfo.Render.Offset;
298 }
299 
resourceCopyBlt(void * sys,void * gpu,uint32_t pitch,uint32_t height,unsigned char upload,ImagePlane plane)300 uint8_t Gmm::resourceCopyBlt(void *sys, void *gpu, uint32_t pitch, uint32_t height, unsigned char upload, ImagePlane plane) {
301     GMM_RES_COPY_BLT gmmResourceCopyBLT = {};
302 
303     if (plane == ImagePlane::PLANE_V) {
304         sys = ptrOffset(sys, height * pitch * 2);
305         pitch /= 2;
306     } else if (plane == ImagePlane::PLANE_U) {
307         sys = ptrOffset(sys, height * pitch * 2 + height * pitch / 2);
308         pitch /= 2;
309     } else if (plane == ImagePlane::PLANE_UV) {
310         sys = ptrOffset(sys, height * pitch * 2);
311     }
312     uint32_t size = pitch * height;
313 
314     gmmResourceCopyBLT.Sys.pData = sys;
315     gmmResourceCopyBLT.Gpu.pData = gpu;
316     gmmResourceCopyBLT.Sys.RowPitch = pitch;
317     gmmResourceCopyBLT.Blt.Upload = upload;
318     gmmResourceCopyBLT.Sys.BufferSize = size;
319 
320     return this->gmmResourceInfo->cpuBlt(&gmmResourceCopyBLT);
321 }
322 
unifiedAuxTranslationCapable() const323 bool Gmm::unifiedAuxTranslationCapable() const {
324     auto gmmFlags = this->gmmResourceInfo->getResourceFlags();
325     UNRECOVERABLE_IF(gmmFlags->Info.RenderCompressed && gmmFlags->Info.MediaCompressed);
326     return gmmFlags->Gpu.CCS && gmmFlags->Gpu.UnifiedAuxSurface && (gmmFlags->Info.RenderCompressed | gmmFlags->Info.MediaCompressed);
327 }
328 
hasMultisampleControlSurface() const329 bool Gmm::hasMultisampleControlSurface() const {
330     return this->gmmResourceInfo->getResourceFlags()->Gpu.MCS;
331 }
332 
getUnifiedAuxPitchTiles()333 uint32_t Gmm::getUnifiedAuxPitchTiles() {
334     return this->gmmResourceInfo->getRenderAuxPitchTiles();
335 }
getAuxQPitch()336 uint32_t Gmm::getAuxQPitch() {
337     return this->gmmResourceInfo->getAuxQPitch();
338 }
339 
applyMemoryFlags(bool systemMemoryPool,StorageInfo & storageInfo)340 void Gmm::applyMemoryFlags(bool systemMemoryPool, StorageInfo &storageInfo) {
341     this->useSystemMemoryPool = systemMemoryPool;
342     auto hardwareInfo = clientContext->getHardwareInfo();
343 
344     if (hardwareInfo->featureTable.flags.ftrLocalMemory) {
345         if (systemMemoryPool) {
346             resourceParams.Flags.Info.NonLocalOnly = 1;
347         } else {
348             if (extraMemoryFlagsRequired()) {
349                 applyExtraMemoryFlags(storageInfo);
350             } else if (!storageInfo.isLockable) {
351                 resourceParams.Flags.Info.NotLockable = 1;
352                 if (isCompressionEnabled || storageInfo.localOnlyRequired) {
353                     resourceParams.Flags.Info.LocalOnly = 1;
354                 }
355             }
356         }
357     }
358 
359     if (hardwareInfo->featureTable.flags.ftrMultiTileArch) {
360         resourceParams.MultiTileArch.Enable = 1;
361         if (systemMemoryPool) {
362             resourceParams.MultiTileArch.GpuVaMappingSet = hardwareInfo->gtSystemInfo.MultiTileArchInfo.TileMask;
363             resourceParams.MultiTileArch.LocalMemPreferredSet = 0;
364             resourceParams.MultiTileArch.LocalMemEligibilitySet = 0;
365 
366         } else {
367             auto tileSelected = std::max(storageInfo.memoryBanks.to_ulong(), 1lu);
368 
369             if (storageInfo.cloningOfPageTables) {
370                 resourceParams.MultiTileArch.GpuVaMappingSet = static_cast<uint8_t>(storageInfo.pageTablesVisibility.to_ulong());
371             } else {
372                 resourceParams.MultiTileArch.TileInstanced = storageInfo.tileInstanced;
373                 resourceParams.MultiTileArch.GpuVaMappingSet = static_cast<uint8_t>(tileSelected);
374             }
375 
376             resourceParams.MultiTileArch.LocalMemPreferredSet = static_cast<uint8_t>(tileSelected);
377             resourceParams.MultiTileArch.LocalMemEligibilitySet = static_cast<uint8_t>(tileSelected);
378         }
379     }
380 }
381 
applyDebugOverrides()382 void Gmm::applyDebugOverrides() {
383     if (-1 != DebugManager.flags.OverrideGmmResourceUsageField.get()) {
384         resourceParams.Usage = static_cast<GMM_RESOURCE_USAGE_TYPE>(DebugManager.flags.OverrideGmmResourceUsageField.get());
385     }
386 }
387 } // namespace NEO
388