1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "Compiler/Optimizer/OCLBIUtils.h"
10 #include "Compiler/CISACodeGen/helper.h"
11 #include "Compiler/MetaDataApi/MetaDataApi.h"
12 #include "Compiler/DebugInfo/Utils.h"
13 #include "common/LLVMWarningsPush.hpp"
14 #include "llvmWrapper/IR/DerivedTypes.h"
15 #include "llvmWrapper/IR/IRBuilder.h"
16 #include "llvmWrapper/IR/Instructions.h"
17 #include "common/LLVMWarningsPop.hpp"
18 #include "LLVM3DBuilder/BuiltinsFrontend.hpp"
19 #include "Probe/Assertion.h"
20 #include "IGC/common/StringMacros.hpp"
21
22 using namespace llvm;
23 using namespace IGC;
24 using namespace IGC::IGCMD;
25
execute(CallInst * Inst,CodeGenContext * CodeGenContext)26 void CCommand::execute(CallInst* Inst, CodeGenContext* CodeGenContext)
27 {
28 init(Inst, CodeGenContext);
29 createIntrinsic();
30 }
31
init(CallInst * Inst,CodeGenContext * CodeGenContext)32 void CCommand::init(CallInst* Inst, CodeGenContext* CodeGenContext)
33 {
34 m_pCallInst = Inst;
35 m_pFunc = m_pCallInst->getParent()->getParent();
36 m_pCtx = &(m_pFunc->getContext());
37 m_pCodeGenContext = CodeGenContext;
38 m_pFloatType = Type::getFloatTy(*m_pCtx);
39 m_pIntType = Type::getInt32Ty(*m_pCtx);
40 m_pFloatZero = Constant::getNullValue(m_pFloatType);
41 m_pIntZero = Constant::getNullValue(m_pIntType);
42 m_pIntOne = ConstantInt::get(m_pIntType, 1);
43 m_DL = m_pCallInst->getDebugLoc();
44 m_args.clear();
45 }
46
getFunctionDeclaration(GenISAIntrinsic::ID id,ArrayRef<Type * > Tys)47 Function* CCommand::getFunctionDeclaration(GenISAIntrinsic::ID id, ArrayRef<Type*> Tys)
48 {
49 return GenISAIntrinsic::getDeclaration(m_pFunc->getParent(), id, Tys);
50 }
51
getFunctionDeclaration(IGCLLVM::Intrinsic id,ArrayRef<Type * > Tys)52 Function* CCommand::getFunctionDeclaration(IGCLLVM::Intrinsic id, ArrayRef<Type*> Tys)
53 {
54 return Intrinsic::getDeclaration(m_pFunc->getParent(), id, Tys);
55 }
56
replaceCallInst(IGCLLVM::Intrinsic intrinsicName,ArrayRef<Type * > Tys)57 void CCommand::replaceCallInst(IGCLLVM::Intrinsic intrinsicName, ArrayRef<Type*> Tys)
58 {
59 Function* func = getFunctionDeclaration(intrinsicName, Tys);
60 Instruction* newCall = CallInst::Create(func, m_args, m_pCallInst->getName(), m_pCallInst);
61
62 if (isa<FPMathOperator>(m_pCallInst)) {
63 if (auto II = dyn_cast<IntrinsicInst>(newCall)) {
64 II->copyFastMathFlags(m_pCallInst->getFastMathFlags());
65 }
66 }
67
68 newCall->setDebugLoc(m_DL);
69 m_pCallInst->replaceAllUsesWith(newCall);
70 }
71
replaceGenISACallInst(GenISAIntrinsic::ID intrinsicName,ArrayRef<Type * > Tys)72 void CCommand::replaceGenISACallInst(GenISAIntrinsic::ID intrinsicName, ArrayRef<Type*> Tys)
73 {
74 Function* func = getFunctionDeclaration(intrinsicName, Tys);
75 Instruction* newCall = CallInst::Create(func, m_args, m_pCallInst->getName(), m_pCallInst);
76 newCall->setDebugLoc(m_DL);
77 m_pCallInst->replaceAllUsesWith(newCall);
78 }
79
prepareZeroOffsets()80 void CImagesBI::prepareZeroOffsets()
81 {
82 m_args.push_back(m_pIntZero); // offsetU
83 m_args.push_back(m_pIntZero); // offsetV
84 m_args.push_back(m_pIntZero); // offsetW
85 }
86
prepareCoords(Dimension Dim,Value * Coord,Value * Zero)87 void CImagesBI::prepareCoords(Dimension Dim, Value* Coord, Value* Zero)
88 {
89 CoordX = Coord;
90 CoordY = Zero;
91 CoordZ = Zero;
92 Instruction* tmp;
93 switch (Dim)
94 {
95 case DIM_3D:
96 case DIM_2D_ARRAY:
97 tmp = ExtractElementInst::Create(
98 Coord,
99 ConstantInt::get(m_pIntType, COORD_Z),
100 "CoordZ",
101 m_pCallInst); // z
102 tmp->setDebugLoc(m_DL);
103 CoordZ = tmp;
104 // fall through
105 case DIM_2D:
106 case DIM_1D_ARRAY:
107 tmp = ExtractElementInst::Create(
108 Coord,
109 ConstantInt::get(m_pIntType, COORD_X),
110 "CoordX",
111 m_pCallInst); // x
112 tmp->setDebugLoc(m_DL);
113 CoordX = tmp;
114 tmp = ExtractElementInst::Create(
115 Coord,
116 ConstantInt::get(m_pIntType, COORD_Y),
117 "CoordY",
118 m_pCallInst); // y
119 tmp->setDebugLoc(m_DL);
120 CoordY = tmp;
121 // fall through
122 case DIM_1D:
123 // no need to extract since in 1 Dim Coord isn't a vector
124 break;
125 }
126 }
127
128
129
prepareColor(Value * Color)130 void CImagesBI::prepareColor(Value* Color)
131 {
132 Value* TmpColor;
133 if (Color->getType()->getScalarType() != m_pFloatType)
134 {
135 // GenISA_typedwrite intrinsic expect to get the color as float,
136 // therefore we do bitcast that should disappear in the final code.
137 Instruction* tmp = BitCastInst::Create(
138 Instruction::BitCast,
139 Color,
140 IGCLLVM::FixedVectorType::get(m_pFloatType, 4),
141 "floatColor",
142 m_pCallInst);
143 tmp->setDebugLoc(m_DL);
144 TmpColor = tmp;
145 }
146 else
147 {
148 TmpColor = Color;
149 }
150
151 Instruction* ColorX = ExtractElementInst::Create(TmpColor, ConstantInt::get(m_pIntType, COORD_X), "ColorX", m_pCallInst); // color x
152 Instruction* ColorY = ExtractElementInst::Create(TmpColor, ConstantInt::get(m_pIntType, COORD_Y), "ColorY", m_pCallInst); // color y
153 Instruction* ColorZ = ExtractElementInst::Create(TmpColor, ConstantInt::get(m_pIntType, COORD_Z), "ColorZ", m_pCallInst); // color z
154 Instruction* ColorW = ExtractElementInst::Create(TmpColor, ConstantInt::get(m_pIntType, COORD_W), "ColorW", m_pCallInst); // color w
155 ColorX->setDebugLoc(m_DL);
156 ColorY->setDebugLoc(m_DL);
157 ColorZ->setDebugLoc(m_DL);
158 ColorW->setDebugLoc(m_DL);
159
160 m_args.push_back(ColorX);
161 m_args.push_back(ColorY);
162 m_args.push_back(ColorZ);
163 m_args.push_back(ColorW);
164
165 }
166
prepareImageBTI()167 void CImagesBI::prepareImageBTI()
168 {
169 Argument* pImg = nullptr;
170 ConstantInt* imageIndex = CImagesUtils::getImageIndex(m_pParamMap, m_pCallInst, 0, pImg);
171 unsigned int addrSpace = EncodeAS4GFXResource(*imageIndex, RESOURCE);
172 Value* img = ConstantPointerNull::get(PointerType::get(pImg->getType(), addrSpace));
173 m_args.push_back(img); // BTI
174
175 // prepareImageBTI() is called for standard image reads. If there is an extension already tagged on
176 // this argument then we have an inconsistent usage of the image and should fail compilation.
177 if (m_pParamMap->count(pImg) != 0)
178 {
179 if ((*m_pParamMap)[pImg].extension != ResourceExtensionTypeEnum::NonExtensionType)
180 {
181 m_IncorrectBti = true;
182 }
183 }
184 }
185
verifyCommand()186 void CImagesBI::verifyCommand()
187 {
188 if (m_IncorrectBti)
189 {
190 m_pCodeGenContext->EmitError("Inconsistent use of image!", NULL);
191 }
192 }
193
findImageFromBufferPtr(const MetaDataUtils & MdUtils,Function * F,BufferType bufType,uint64_t idx,const IGC::ModuleMetaData * modMD)194 Argument* CImagesBI::CImagesUtils::findImageFromBufferPtr(const MetaDataUtils& MdUtils, Function* F, BufferType bufType, uint64_t idx, const IGC::ModuleMetaData* modMD)
195 {
196 if (modMD->FuncMD.find(F) != modMD->FuncMD.end())
197 {
198 FunctionMetaData funcMD = modMD->FuncMD.find(F)->second;
199 ResourceAllocMD resAllocMD = funcMD.resAllocMD;
200 IGC_ASSERT_MESSAGE(resAllocMD.argAllocMDList.size() > 0, "ArgAllocMDList is empty.");
201 for (auto& arg : F->args())
202 {
203 unsigned argNo = arg.getArgNo();
204 ArgAllocMD argAlloc = resAllocMD.argAllocMDList[argNo];
205 ResourceTypeEnum argType = (ResourceTypeEnum)argAlloc.type;
206 if (ResourceTypeMap(argType) == bufType &&
207 argAlloc.indexType == idx)
208 return &arg;
209 }
210 }
211
212 return nullptr;
213 }
214
getImageIndex(ParamMap * pParamMap,CallInst * pCallInst,unsigned int paramIndex,Argument * & imageParam)215 ConstantInt* CImagesBI::CImagesUtils::getImageIndex(
216 ParamMap* pParamMap,
217 CallInst* pCallInst,
218 unsigned int paramIndex,
219 Argument*& imageParam)
220 {
221 ConstantInt* imageIndex = nullptr;
222 imageParam = cast<Argument>(ValueTracker::track(pCallInst, paramIndex));
223 int i = (*pParamMap)[imageParam].index;
224 imageIndex = ConstantInt::get(Type::getInt32Ty(pCallInst->getContext()), i);
225 return imageIndex;
226 }
227
getImageType(ParamMap * pParamMap,CallInst * pCallInst,unsigned int paramIndex)228 BufferType CImagesBI::CImagesUtils::getImageType(ParamMap* pParamMap, CallInst* pCallInst, unsigned int paramIndex)
229 {
230 Argument* imageParam = cast<Argument>(ValueTracker::track(pCallInst, paramIndex));
231 return (*pParamMap)[imageParam].type;
232 }
233
createGetBufferPtr()234 void CImagesBI::createGetBufferPtr()
235 {
236 Argument* pImg = nullptr;
237 ConstantInt* imageIndex = CImagesUtils::getImageIndex(m_pParamMap, m_pCallInst, 0, pImg);
238 BufferType bufType = CImagesUtils::getImageType(m_pParamMap, m_pCallInst, 0);
239 unsigned int addressSpace = IGC::EncodeAS4GFXResource(*imageIndex, bufType);
240 Type* ptrTy = llvm::PointerType::get(m_pFloatType, addressSpace);
241
242 Function* pFuncGetBufferPtr = getFunctionDeclaration(GenISAIntrinsic::GenISA_GetBufferPtr, ptrTy);
243
244 //%base_ptr = call float* @llvm.GenISA.GetBufferPtr(i32 %bufIdx, i32 %type)
245
246 // preparing the argumant list for the function
247 llvm::SmallVector<Value*, 2> getBufferPtrArgs;
248 getBufferPtrArgs.push_back(imageIndex);
249 getBufferPtrArgs.push_back(ConstantInt::get(m_pIntType, bufType));
250
251 CallInst* pDstBuffer = CallInst::Create(
252 pFuncGetBufferPtr,
253 getBufferPtrArgs,
254 m_pCallInst->getName(),
255 m_pCallInst);
256 pDstBuffer->setDebugLoc(m_DL);
257 m_args.push_back(pDstBuffer);
258
259 // createGetBufferPtr() is called for standard writes reads. If there is an extension already tagged on
260 // this argument then we have an inconsistent usage of the image and should fail compilation.
261 if (m_pParamMap->count(pImg) != 0)
262 {
263 if ((*m_pParamMap)[pImg].extension != ResourceExtensionTypeEnum::NonExtensionType)
264 {
265 m_IncorrectBti = true;
266 }
267 }
268 }
269
isInteger(APFloat & val)270 inline bool isInteger(APFloat& val)
271 {
272 // This could be made more efficient; I'm going for obviously correct.
273 if (!val.isFinite())
274 return false;
275 APFloat truncated = val;
276 truncated.roundToIntegral(APFloat::rmTowardZero);
277 return val.compare(truncated) == APFloat::cmpEqual;
278 }
279
derivedFromInt(const Value * pVal)280 bool CImagesBI::derivedFromInt(const Value* pVal)
281 {
282 SmallPtrSet<const Value*, 16> visitedPHIs;
283
284 std::function<bool(const Value*)> go = [&](const Value* val) -> bool
285 {
286 if (val->getType()->getScalarType()->isIntegerTy() || isa<UndefValue>(val))
287 {
288 return true;
289 }
290 else if (auto * constVal = dyn_cast<ConstantFP>(val))
291 {
292 APFloat val = constVal->getValueAPF();
293 return isInteger(val);
294 }
295 else if (auto * inst = dyn_cast<Instruction>(val))
296 {
297 switch (inst->getOpcode())
298 {
299 case Instruction::PHI:
300 {
301 // return "true" if we have visited this PHI before.
302 for (const Value* phi : visitedPHIs)
303 {
304 if (phi == inst)
305 return true;
306 }
307 visitedPHIs.insert(inst);
308 for (uint i = 0, numOpnds = inst->getNumOperands(); i < numOpnds; i++)
309 {
310 if (!go(inst->getOperand(i)))
311 return false;
312 }
313 visitedPHIs.erase(inst);
314 return true;
315 }
316
317 case Instruction::Select:
318 {
319 auto* sel = cast<SelectInst>(inst);
320 return (go(sel->getTrueValue()) && go(sel->getFalseValue()));
321 }
322
323 case Instruction::InsertElement:
324 {
325 return (go(inst->getOperand(0)) && go(inst->getOperand(1)));
326 }
327
328 case Instruction::ExtractElement:
329 {
330 auto* extrElem = cast<ExtractElementInst>(inst);
331 return (go(extrElem->getVectorOperand()));
332 }
333
334 case Instruction::UIToFP:
335 case Instruction::SIToFP:
336 return true;
337
338 default:
339 return false;
340 }
341 }
342 else
343 {
344 return false;
345 }
346 };
347
348 return go(pVal);
349 }
350
351 class COCL_sample : public CImagesBI
352 {
353 public:
COCL_sample(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim,MetaDataUtils * pMdUtils,ModuleMetaData * modMD)354 COCL_sample(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim, MetaDataUtils* pMdUtils, ModuleMetaData* modMD) : CImagesBI(paramMap, inlineMap, nextSampler, Dim), m_pMdUtils(pMdUtils), m_modMD(modMD) {}
355
getSamplerIndex(void)356 ConstantInt* getSamplerIndex(void)
357 {
358 ConstantInt* samplerIndex = nullptr;
359 Value* samplerParam = ValueTracker::track(m_pCallInst, 1, m_pMdUtils, m_modMD);
360 if (!samplerParam) {
361 m_pCodeGenContext->EmitError("There are instructions that use a sampler, but no sampler found in the kernel!", m_pCallInst);
362 return nullptr;
363 }
364
365 // Argument samplers are looked up in the parameter map
366 if (isa<Argument>(samplerParam))
367 {
368 int i = (*m_pParamMap)[samplerParam].index;
369 samplerIndex = ConstantInt::get(m_pIntType, i);
370 return samplerIndex;
371 }
372
373 // The sampler is not an argument, make sure it's a constant
374 IGC_ASSERT_MESSAGE(isa<ConstantInt>(samplerParam), "Sampler must be a global variable or a constant");
375 ConstantInt* constSampler = cast<ConstantInt>(samplerParam);
376 int samplerValue = int_cast<int>(constSampler->getZExtValue());
377
378 // Is this sampler already allocated?
379 auto iter = m_pInlineMap->find(samplerValue);
380 if (iter != m_pInlineMap->end())
381 {
382 return ConstantInt::get(m_pIntType, iter->second);
383 }
384
385 // No, allocate it.
386 int currSamplerIdx = (*m_pNextSampler)++;
387 (*m_pInlineMap)[samplerValue] = currSamplerIdx;
388 samplerIndex = ConstantInt::get(m_pIntType, currSamplerIdx);
389
390 // Push this information into the metadata, for the state processor's benefit
391 FunctionMetaData& funcMD = m_modMD->FuncMD[m_pFunc];
392 ResourceAllocMD& resAllocMD = funcMD.resAllocMD;
393 InlineSamplersMD inlineSamplerMD;
394 CreateInlineSamplerAnnotations(inlineSamplerMD, samplerValue);
395 inlineSamplerMD.index = currSamplerIdx;
396 resAllocMD.inlineSamplersMD.push_back(inlineSamplerMD);
397 m_pMdUtils->save(*m_pCtx);
398 return samplerIndex;
399 }
400
CreateInlineSamplerAnnotations(InlineSamplersMD & inlineSamplerMD,int samplerValue)401 void CreateInlineSamplerAnnotations(InlineSamplersMD& inlineSamplerMD, int samplerValue)
402 {
403 inlineSamplerMD.m_Value = samplerValue;
404 Module* M = m_pCallInst->getParent()->getModule();
405 if (llvm::StringRef(M->getTargetTriple()).startswith("igil") || llvm::StringRef(M->getTargetTriple()).startswith("gpu_64"))
406 {
407 inlineSamplerMD.addressMode = samplerValue & LEGACY_SAMPLER_ADDRESS_MASK;
408 switch (samplerValue & LEGACY_SAMPLER_ADDRESS_MASK)
409 {
410 case LEGACY_CLK_ADDRESS_NONE:
411 inlineSamplerMD.TCXAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP;
412 inlineSamplerMD.TCYAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP;
413 inlineSamplerMD.TCZAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP;
414 break;
415 case LEGACY_CLK_ADDRESS_CLAMP:
416 inlineSamplerMD.TCXAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_BORDER;
417 inlineSamplerMD.TCYAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_BORDER;
418 inlineSamplerMD.TCZAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_BORDER;
419 break;
420 case LEGACY_CLK_ADDRESS_CLAMP_TO_EDGE:
421 inlineSamplerMD.TCXAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP;
422 inlineSamplerMD.TCYAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP;
423 inlineSamplerMD.TCZAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP;
424 break;
425 case LEGACY_CLK_ADDRESS_REPEAT:
426 inlineSamplerMD.TCXAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_WRAP;
427 inlineSamplerMD.TCYAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_WRAP;
428 inlineSamplerMD.TCZAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_WRAP;
429 break;
430 case LEGACY_CLK_ADDRESS_MIRRORED_REPEAT:
431 inlineSamplerMD.TCXAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR;
432 inlineSamplerMD.TCYAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR;
433 inlineSamplerMD.TCZAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR;
434 break;
435 #if defined ( _DEBUG ) || defined ( _INTERNAL )
436 case LEGACY_CLK_ADDRESS_MIRRORED_REPEAT_101_INTEL:
437 inlineSamplerMD.TCXAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR101;
438 inlineSamplerMD.TCYAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR101;
439 inlineSamplerMD.TCZAddressMode = iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR101;
440 break;
441 #endif // RELEASE INTERNAL||DEBUG
442
443 default:
444 IGC_ASSERT_MESSAGE(0, "Invalid sampler type");
445 break;
446 }
447
448 inlineSamplerMD.NormalizedCoords = (samplerValue & LEGACY_SAMPLER_NORMALIZED_MASK);
449
450 switch (samplerValue & LEGACY_SAMPLER_FILTER_MASK)
451 {
452 case LEGACY_CLK_FILTER_NEAREST:
453 inlineSamplerMD.MagFilterType = (iOpenCL::SAMPLER_MAPFILTER_POINT);
454 inlineSamplerMD.MinFilterType = (iOpenCL::SAMPLER_MAPFILTER_POINT);
455 break;
456 case LEGACY_CLK_FILTER_LINEAR:
457 inlineSamplerMD.MagFilterType = (iOpenCL::SAMPLER_MAPFILTER_LINEAR);
458 inlineSamplerMD.MinFilterType = (iOpenCL::SAMPLER_MAPFILTER_LINEAR);
459 break;
460 default:
461 IGC_ASSERT_MESSAGE(0, "Filter Type must have value");
462 break;
463 }
464
465 // Border color should always be transparent black:
466 inlineSamplerMD.BorderColorR = (0.0f);
467 inlineSamplerMD.BorderColorG = (0.0f);
468 inlineSamplerMD.BorderColorB = (0.0f);
469 inlineSamplerMD.BorderColorA = (0.0f);
470 }
471 else if (llvm::StringRef(M->getTargetTriple()).startswith("spir"))
472 {
473 switch (samplerValue & SPIR_SAMPLER_ADDRESS_MASK)
474 {
475 case SPIR_CLK_ADDRESS_NONE:
476 inlineSamplerMD.TCXAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP);
477 inlineSamplerMD.TCYAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP);
478 inlineSamplerMD.TCZAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP);
479 inlineSamplerMD.addressMode = (LEGACY_CLK_ADDRESS_NONE);
480 break;
481 case SPIR_CLK_ADDRESS_CLAMP:
482 inlineSamplerMD.TCXAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_BORDER);
483 inlineSamplerMD.TCYAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_BORDER);
484 inlineSamplerMD.TCZAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_BORDER);
485 inlineSamplerMD.addressMode = (LEGACY_CLK_ADDRESS_CLAMP);
486 break;
487 case SPIR_CLK_ADDRESS_CLAMP_TO_EDGE:
488 inlineSamplerMD.TCXAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP);
489 inlineSamplerMD.TCYAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP);
490 inlineSamplerMD.TCZAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_CLAMP);
491 inlineSamplerMD.addressMode = (LEGACY_CLK_ADDRESS_CLAMP_TO_EDGE);
492 break;
493 case SPIR_CLK_ADDRESS_REPEAT:
494 inlineSamplerMD.TCXAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_WRAP);
495 inlineSamplerMD.TCYAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_WRAP);
496 inlineSamplerMD.TCZAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_WRAP);
497 inlineSamplerMD.addressMode = (LEGACY_CLK_ADDRESS_REPEAT);
498 break;
499 case SPIR_CLK_ADDRESS_MIRRORED_REPEAT:
500 inlineSamplerMD.TCXAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR);
501 inlineSamplerMD.TCYAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR);
502 inlineSamplerMD.TCZAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR);
503 inlineSamplerMD.addressMode = (LEGACY_CLK_ADDRESS_MIRRORED_REPEAT);
504 break;
505 #if defined ( _DEBUG ) || defined ( _INTERNAL )
506 case SPIR_CLK_ADDRESS_MIRRORED_REPEAT_101_INTEL:
507 inlineSamplerMD.TCXAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR101);
508 inlineSamplerMD.TCYAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR101);
509 inlineSamplerMD.TCZAddressMode = (iOpenCL::SAMPLER_TEXTURE_ADDRESS_MODE_MIRROR101);
510 break;
511 #endif // RELEASE INTERNAL||DEBUG
512
513 default:
514 IGC_ASSERT_MESSAGE(0, "Invalid sampler type");
515 break;
516 }
517
518 switch (samplerValue & SPIR_SAMPLER_NORMALIZED_MASK)
519 {
520 case SPIR_CLK_NORMALIZED_COORDS_TRUE:
521 inlineSamplerMD.NormalizedCoords = (LEGACY_CLK_NORMALIZED_COORDS_TRUE);
522 break;
523 case SPIR_CLK_NORMALIZED_COORDS_FALSE:
524 inlineSamplerMD.NormalizedCoords = (LEGACY_CLK_NORMALIZED_COORDS_FALSE);
525 break;
526 default:
527 IGC_ASSERT_MESSAGE(0, "Invalid normalized coords");
528 break;
529 }
530
531
532 switch (samplerValue & SPIR_SAMPLER_FILTER_MASK)
533 {
534 case SPIR_CLK_FILTER_NEAREST:
535 inlineSamplerMD.MagFilterType = (iOpenCL::SAMPLER_MAPFILTER_POINT);
536 inlineSamplerMD.MinFilterType = (iOpenCL::SAMPLER_MAPFILTER_POINT);
537 break;
538 case SPIR_CLK_FILTER_LINEAR:
539 inlineSamplerMD.MagFilterType = (iOpenCL::SAMPLER_MAPFILTER_LINEAR);
540 inlineSamplerMD.MinFilterType = (iOpenCL::SAMPLER_MAPFILTER_LINEAR);
541 break;
542 default:
543 IGC_ASSERT_MESSAGE(0, "Filter Type must have value");
544 break;
545 }
546
547 // Border color should always be transparent black:
548 inlineSamplerMD.BorderColorR = (0.0f);
549 inlineSamplerMD.BorderColorG = (0.0f);
550 inlineSamplerMD.BorderColorB = (0.0f);
551 inlineSamplerMD.BorderColorA = (0.0f);
552 }
553 else
554 {
555 IGC_ASSERT_MESSAGE(0, "Input IR version must be OCL or SPIR");
556 }
557 }
558
prepareSamplerIndex()559 bool prepareSamplerIndex()
560 {
561 ConstantInt* samplerIndex = getSamplerIndex();
562 if (!samplerIndex) return false;
563 unsigned int addrSpace = EncodeAS4GFXResource(*samplerIndex, SAMPLER);
564 Value* sampler = ConstantPointerNull::get(PointerType::get(samplerIndex->getType(), addrSpace));
565 m_args.push_back(sampler);
566 return true;
567 }
568
prepareGradients(Dimension Dim,Value * gradX,Value * gradY)569 void prepareGradients(Dimension Dim, Value* gradX, Value* gradY)
570 {
571 m_gradXY = m_gradXZ = m_gradYY = m_gradYZ = m_pFloatZero;
572 m_gradXX = gradX;
573 m_gradYX = gradY;
574 Instruction* tmp;
575 switch (Dim)
576 {
577 case DIM_3D:
578 tmp = ExtractElementInst::Create(
579 gradX,
580 ConstantInt::get(m_pIntType, COORD_Z),
581 "gradXZ",
582 m_pCallInst);
583 tmp->setDebugLoc(m_DL);
584 m_gradXZ = tmp;
585 tmp = ExtractElementInst::Create(
586 gradY,
587 ConstantInt::get(m_pIntType, COORD_Z),
588 "gradYZ",
589 m_pCallInst);
590 tmp->setDebugLoc(m_DL);
591 m_gradYZ = tmp;
592 // fall through
593 case DIM_2D:
594 case DIM_2D_ARRAY:
595 tmp = ExtractElementInst::Create(
596 gradX,
597 ConstantInt::get(m_pIntType, COORD_Y),
598 "gradXY",
599 m_pCallInst);
600 tmp->setDebugLoc(m_DL);
601 m_gradXY = tmp;
602 tmp = ExtractElementInst::Create(
603 gradY,
604 ConstantInt::get(m_pIntType, COORD_Y),
605 "gradYY",
606 m_pCallInst);
607 tmp->setDebugLoc(m_DL);
608 m_gradYY = tmp;
609 tmp = ExtractElementInst::Create(
610 gradX,
611 ConstantInt::get(m_pIntType, COORD_X),
612 "gradXX",
613 m_pCallInst);
614 tmp->setDebugLoc(m_DL);
615 m_gradXX = tmp;
616 tmp = ExtractElementInst::Create(
617 gradY,
618 ConstantInt::get(m_pIntType, COORD_X),
619 "gradYX",
620 m_pCallInst);
621 tmp->setDebugLoc(m_DL);
622 m_gradYX = tmp;
623 // fall through
624 case DIM_1D:
625 case DIM_1D_ARRAY:
626 // no need to extract since in 1 Dim gradient is not a vector.
627 break;
628 }
629 }
630
631 protected:
632 MetaDataUtils* m_pMdUtils;
633 ModuleMetaData* m_modMD;
634 Value* m_gradXX;
635 Value* m_gradXY;
636 Value* m_gradXZ;
637 Value* m_gradYX;
638 Value* m_gradYY;
639 Value* m_gradYZ;
640 };
641
642 class COCL_sample_l : public COCL_sample
643 {
644 public:
COCL_sample_l(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim,MetaDataUtils * pMdUtils,ModuleMetaData * modMD)645 COCL_sample_l(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim, MetaDataUtils* pMdUtils, ModuleMetaData* modMD) : COCL_sample(paramMap, inlineMap, nextSampler, Dim, pMdUtils, modMD) {}
646
createIntrinsic()647 void createIntrinsic()
648 {
649 Value* Coord = m_pCallInst->getOperand(2);
650 m_args.push_back(m_pCallInst->getOperand(3)); // lod
651 prepareCoords(m_dim, Coord, m_pFloatZero);
652 m_args.push_back(CoordX);
653 m_args.push_back(CoordY);
654 m_args.push_back(CoordZ);
655 m_args.push_back(m_pFloatZero); // ai (?)
656 createGetBufferPtr();
657 bool samplerIndexFound = prepareSamplerIndex();
658 if (!samplerIndexFound) return;
659
660 prepareZeroOffsets();
661 Type* types[] = {
662 m_pCallInst->getType(),
663 m_pFloatType,
664 m_args[5]->getType(),
665 m_args[6]->getType(),
666 };
667 replaceGenISACallInst(GenISAIntrinsic::GenISA_sampleLptr, types);
668 }
669 };
670
671 class COCL_sample_d : public COCL_sample
672 {
673 public:
COCL_sample_d(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim,MetaDataUtils * pMdUtils,ModuleMetaData * modMD)674 COCL_sample_d(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim, MetaDataUtils* pMdUtils, ModuleMetaData* modMD) : COCL_sample(paramMap, inlineMap, nextSampler, Dim, pMdUtils, modMD) {}
675
createIntrinsic()676 void createIntrinsic()
677 {
678 Value* Coord = m_pCallInst->getOperand(2);
679 prepareCoords(m_dim, Coord, m_pFloatZero);
680 prepareGradients(m_dim, m_pCallInst->getOperand(3), m_pCallInst->getOperand(4));
681 m_args.push_back(CoordX);
682 m_args.push_back(m_gradXX);
683 m_args.push_back(m_gradYX);
684 m_args.push_back(CoordY);
685 m_args.push_back(m_gradXY);
686 m_args.push_back(m_gradYY);
687 m_args.push_back(CoordZ);
688 m_args.push_back(m_gradXZ);
689 m_args.push_back(m_gradYZ);
690 m_args.push_back(m_pFloatZero); // ai (?)
691 m_args.push_back(m_pFloatZero); // minLOD (?)
692 prepareImageBTI();
693 prepareSamplerIndex();
694 prepareZeroOffsets();
695 Type* types[] = {
696 m_pCallInst->getType(),
697 m_pFloatType,
698 m_args[11]->getType(),
699 m_args[12]->getType(),
700 };
701 replaceGenISACallInst(GenISAIntrinsic::GenISA_sampleDptr, types);
702 }
703 };
704
705 class COCL_ldui : public CImagesBI
706 {
707 public:
COCL_ldui(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)708 COCL_ldui(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : CImagesBI(paramMap, inlineMap, nextSampler, Dim) {}
709
createIntrinsic()710 void createIntrinsic()
711 {
712 Value* Coord = m_pCallInst->getOperand(1);
713 prepareCoords(m_dim, Coord, m_pIntZero);
714 m_args.push_back(CoordX);
715 m_args.push_back(CoordY);
716 m_args.push_back(m_pCallInst->getOperand(2)); // LOD
717 m_args.push_back(CoordZ);
718 createGetBufferPtr();
719 prepareZeroOffsets();
720 Type* types[] = {
721 m_pCallInst->getType(),
722 m_args[4]->getType(),
723 };
724 replaceGenISACallInst(GenISAIntrinsic::GenISA_ldptr, types);
725 }
726 };
727
728 class COCL_ld : public CImagesBI
729 {
730 public:
COCL_ld(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)731 COCL_ld(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : CImagesBI(paramMap, inlineMap, nextSampler, Dim) {}
732
createIntrinsic()733 void createIntrinsic()
734 {
735 Value* Coord = m_pCallInst->getOperand(1);
736 prepareCoords(m_dim, Coord, m_pIntZero);
737 m_args.push_back(CoordX);
738 m_args.push_back(CoordY);
739 m_args.push_back(m_pCallInst->getOperand(2)); // LOD
740 m_args.push_back(CoordZ);
741 createGetBufferPtr();
742 prepareZeroOffsets();
743 Type* types[] = {
744 IGCLLVM::FixedVectorType::get(m_pFloatType, 4),
745 m_args[4]->getType(),
746 };
747 replaceGenISACallInst(GenISAIntrinsic::GenISA_ldptr, types);
748 }
749 };
750
751 class COCL_ldmcs : public CImagesBI
752 {
753 public:
COCL_ldmcs(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)754 COCL_ldmcs(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : CImagesBI(paramMap, inlineMap, nextSampler, Dim) {}
755
createIntrinsic()756 void createIntrinsic()
757 {
758 Value* Coord = m_pCallInst->getOperand(1);
759 prepareCoords(m_dim, Coord, m_pIntZero);
760 m_args.push_back(CoordX);
761 m_args.push_back(CoordY);
762 m_args.push_back(CoordZ);
763 m_args.push_back(m_pIntZero); // LOD
764 prepareImageBTI();
765 prepareZeroOffsets();
766 Type* types[] = { IGCLLVM::FixedVectorType::get(m_pFloatType, 4), m_pIntType, m_args[7]->getType() };
767 replaceGenISACallInst(GenISAIntrinsic::GenISA_ldmcsptr, types);
768 }
769 };
770
771 class COCL_ld2dms_base : public CImagesBI
772 {
773 public:
COCL_ld2dms_base(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)774 COCL_ld2dms_base(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : CImagesBI(paramMap, inlineMap, nextSampler, Dim) {}
775
776 protected:
777
778 // adds mcs information to the arguments.
779 // mcs is received as a float vector, from which the 2 lower element need
780 // to be extracted, bitcast to i32, and then pushed to the arguments list.
prepareMCS(Value * mcs)781 void prepareMCS(Value* mcs)
782 {
783 Instruction* fmcsl = ExtractElementInst::Create(mcs, m_pIntZero, "mcsl", m_pCallInst);
784 Instruction* fmcsh = ExtractElementInst::Create(mcs, m_pIntOne, "mcsh", m_pCallInst);
785 Instruction* imcsl = BitCastInst::Create(Instruction::BitCast, fmcsl, m_pIntType, "imcsl", m_pCallInst);
786 Instruction* imcsh = BitCastInst::Create(Instruction::BitCast, fmcsh, m_pIntType, "imcsh", m_pCallInst);
787 fmcsl->setDebugLoc(m_DL);
788 fmcsh->setDebugLoc(m_DL);
789 imcsl->setDebugLoc(m_DL);
790 imcsh->setDebugLoc(m_DL);
791
792 m_args.push_back(imcsl);
793 m_args.push_back(imcsh);
794 }
795 };
796
797 class COCL_ld2dms : public COCL_ld2dms_base
798 {
799 public:
COCL_ld2dms(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)800 COCL_ld2dms(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : COCL_ld2dms_base(paramMap, inlineMap, nextSampler, Dim) {}
801
createIntrinsic()802 void createIntrinsic()
803 {
804 Value* Coord = m_pCallInst->getOperand(1);
805 prepareCoords(m_dim, Coord, m_pIntZero);
806 m_args.push_back(m_pCallInst->getOperand(2)); // sample index
807 prepareMCS(m_pCallInst->getOperand(3));
808 m_args.push_back(CoordX);
809 m_args.push_back(CoordY);
810 m_args.push_back(CoordZ);
811 m_args.push_back(m_pIntZero); // LOD
812 prepareImageBTI();
813 prepareZeroOffsets();
814 replaceGenISACallInst(GenISAIntrinsic::GenISA_ldmsptr, { IGCLLVM::FixedVectorType::get(m_pIntType, 4), m_args[7]->getType() });
815 }
816 };
817
818 class COCL_ld2dmsui : public COCL_ld2dms_base
819 {
820 public:
COCL_ld2dmsui(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)821 COCL_ld2dmsui(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : COCL_ld2dms_base(paramMap, inlineMap, nextSampler, Dim) {}
822
createIntrinsic()823 void createIntrinsic()
824 {
825 Value* Coord = m_pCallInst->getOperand(1);
826 prepareCoords(m_dim, Coord, m_pIntZero);
827 m_args.push_back(m_pCallInst->getOperand(2)); // sample index
828 prepareMCS(m_pCallInst->getOperand(3));
829 m_args.push_back(CoordX);
830 m_args.push_back(CoordY);
831 m_args.push_back(CoordZ);
832 m_args.push_back(m_pIntZero); // LOD
833 prepareImageBTI();
834 prepareZeroOffsets();
835 replaceGenISACallInst(GenISAIntrinsic::GenISA_ldmsptr, { IGCLLVM::FixedVectorType::get(m_pIntType, 4), m_args[7]->getType() });
836 }
837 };
838
839 class CWrite : public CImagesBI
840 {
841 public:
CWrite(ParamMap * paramMap,InlineMap * inlineMap,int * nextSampler,Dimension Dim)842 CWrite(ParamMap* paramMap, InlineMap* inlineMap, int* nextSampler, Dimension Dim) : CImagesBI(paramMap, inlineMap, nextSampler, Dim) {}
843
createIntrinsic()844 void createIntrinsic()
845 {
846 Value* Coord = m_pCallInst->getOperand(1);
847 Value* Color = m_pCallInst->getOperand(2);
848 createGetBufferPtr();
849 prepareCoords(m_dim, Coord, m_pIntZero);
850 m_args.push_back(CoordX);
851 m_args.push_back(CoordY);
852 m_args.push_back(CoordZ);
853 m_args.push_back(m_pCallInst->getOperand(3)); // LOD
854 prepareColor(Color);
855 replaceGenISACallInst(GenISAIntrinsic::GenISA_typedwrite, llvm::ArrayRef<llvm::Type*>(m_args[0]->getType()));
856 }
857 };
858
859 class CSimpleIntrinMapping : public CCommand
860 {
861 // id - ID of the intrinsic to be replaced by the call
862 const GenISAIntrinsic::ID isaId;
863
864 const IGCLLVM::Intrinsic id;
865 // isOverloadable - true if the mapped intrinsic is over-loadable
866 const bool isOverloadable;
867 protected:
createIntrinsicType(const CallInst * pCI,ArrayRef<Type * > overloadTypes)868 void createIntrinsicType(const CallInst* pCI, ArrayRef<Type*> overloadTypes)
869 {
870 m_args.append(pCI->op_begin(), pCI->op_begin() + pCI->getNumArgOperands());
871 IGC_ASSERT_MESSAGE(!(id != Intrinsic::num_intrinsics && isaId != GenISAIntrinsic::ID::num_genisa_intrinsics), "Both intrinsic id's cannot be valid at the same time");
872
873 // GenISA intrinsics ID start after llvm intrinsics
874 if (isaId != GenISAIntrinsic::num_genisa_intrinsics && isaId > static_cast<int>(Intrinsic::num_intrinsics))
875 {
876 replaceGenISACallInst(isaId, this->isOverloadable ? overloadTypes : None);
877 }
878 else
879 {
880 replaceCallInst(id, this->isOverloadable ? overloadTypes : None);
881 }
882 }
883 public:
CSimpleIntrinMapping(GenISAIntrinsic::ID intrinsicId,bool isOverloadable)884 CSimpleIntrinMapping(GenISAIntrinsic::ID intrinsicId, bool isOverloadable)
885 : isaId(intrinsicId), id(Intrinsic::num_intrinsics), isOverloadable(isOverloadable) {}
886
CSimpleIntrinMapping(IGCLLVM::Intrinsic intrinsicId,bool isOverloadable)887 CSimpleIntrinMapping(IGCLLVM::Intrinsic intrinsicId, bool isOverloadable)
888 : isaId(GenISAIntrinsic::ID::num_genisa_intrinsics), id(intrinsicId), isOverloadable(isOverloadable)
889 {}
890
create(IGCLLVM::Intrinsic intrinsicId,bool isOverloadable=true)891 static std::unique_ptr<CSimpleIntrinMapping> create(IGCLLVM::Intrinsic intrinsicId, bool isOverloadable = true)
892 {
893 return std::unique_ptr<CSimpleIntrinMapping>(new CSimpleIntrinMapping(intrinsicId, isOverloadable));
894 }
895
create(GenISAIntrinsic::ID intrinsicId,bool isOverloadable=true)896 static std::unique_ptr<CSimpleIntrinMapping> create(GenISAIntrinsic::ID intrinsicId, bool isOverloadable = true)
897 {
898 return std::unique_ptr<CSimpleIntrinMapping>(new CSimpleIntrinMapping(intrinsicId, isOverloadable));
899 }
900
createIntrinsic()901 void createIntrinsic()
902 {
903 IGC_ASSERT_MESSAGE(!(this->isOverloadable && m_pCallInst->getNumArgOperands() == 0), "Cannot create an overloadable with no args");
904 llvm::Type* tys[2];
905 switch (isaId)
906 {
907 case GenISAIntrinsic::GenISA_ftoi_rtn:
908 case GenISAIntrinsic::GenISA_ftoi_rtp:
909 case GenISAIntrinsic::GenISA_ftoi_rte:
910 case GenISAIntrinsic::GenISA_ftoui_rtn:
911 case GenISAIntrinsic::GenISA_ftoui_rtp:
912 case GenISAIntrinsic::GenISA_ftoui_rte:
913 case GenISAIntrinsic::GenISA_ftof_rtn:
914 case GenISAIntrinsic::GenISA_ftof_rtp:
915 case GenISAIntrinsic::GenISA_ftof_rtz:
916 case GenISAIntrinsic::GenISA_uitof_rtn:
917 case GenISAIntrinsic::GenISA_uitof_rtp:
918 case GenISAIntrinsic::GenISA_uitof_rtz:
919 case GenISAIntrinsic::GenISA_itof_rtn:
920 case GenISAIntrinsic::GenISA_itof_rtp:
921 case GenISAIntrinsic::GenISA_itof_rtz:
922 tys[0] = m_pCallInst->getCalledFunction()->getReturnType();
923 tys[1] = m_pCallInst->getArgOperand(0)->getType();
924 m_args.append(m_pCallInst->op_begin(), m_pCallInst->op_begin() + m_pCallInst->getNumArgOperands());
925 IGC_ASSERT_MESSAGE(!(id != Intrinsic::num_intrinsics && isaId != GenISAIntrinsic::ID::num_genisa_intrinsics), "Both intrinsic id's cannot be valid at the same time");
926 replaceGenISACallInst(isaId, llvm::ArrayRef<llvm::Type*>(tys));
927 break;
928 case GenISAIntrinsic::GenISA_simdGetMessagePhase:
929 case GenISAIntrinsic::GenISA_broadcastMessagePhase:
930 case GenISAIntrinsic::GenISA_createMessagePhasesV:
931 case GenISAIntrinsic::GenISA_createMessagePhasesNoInitV:
932 case GenISAIntrinsic::GenISA_getMessagePhaseX:
933 case GenISAIntrinsic::GenISA_setMessagePhaseV:
934 createIntrinsicType(m_pCallInst, m_pCallInst->getCalledFunction()->getReturnType());
935 break;
936 case GenISAIntrinsic::GenISA_simdSetMessagePhase:
937 case GenISAIntrinsic::GenISA_setMessagePhaseX:
938 case GenISAIntrinsic::GenISA_setMessagePhaseX_legacy:
939 createIntrinsicType(m_pCallInst, m_pCallInst->getArgOperand(m_pCallInst->getNumArgOperands() - 1)->getType());
940 break;
941 case GenISAIntrinsic::GenISA_broadcastMessagePhaseV:
942 case GenISAIntrinsic::GenISA_simdGetMessagePhaseV:
943 case GenISAIntrinsic::GenISA_getMessagePhaseXV:
944 {
945 Type* overloadTypes[] =
946 {
947 m_pCallInst->getCalledFunction()->getReturnType(),
948 m_pCallInst->getArgOperand(0)->getType()
949 };
950 createIntrinsicType(m_pCallInst, overloadTypes);
951 break;
952 }
953 case GenISAIntrinsic::GenISA_simdSetMessagePhaseV:
954 case GenISAIntrinsic::GenISA_setMessagePhaseXV:
955 {
956 Type* overloadTypes[] =
957 {
958 m_pCallInst->getCalledFunction()->getReturnType(),
959 m_pCallInst->getArgOperand(m_pCallInst->getNumArgOperands() - 1)->getType()
960 };
961 createIntrinsicType(m_pCallInst, overloadTypes);
962 break;
963 }
964 default:
965 {
966 auto* pOverloadType = (!this->isOverloadable) ?
967 nullptr :
968 m_pCallInst->getArgOperand(0)->getType();
969 createIntrinsicType(m_pCallInst, pOverloadType);
970 break;
971 }
972 }
973 }
974 };
975
976 class CWaveBallotIntrinsic : public CCommand
977 {
978 private:
979 // id - ID of the intrinsic to be replaced by the call
980 const GenISAIntrinsic::ID isaId;
981 const IGCLLVM::Intrinsic id;
982
983 public:
CWaveBallotIntrinsic(GenISAIntrinsic::ID intrinsicId)984 CWaveBallotIntrinsic(GenISAIntrinsic::ID intrinsicId)
985 : isaId(intrinsicId), id(Intrinsic::num_intrinsics) {}
986
CWaveBallotIntrinsic(IGCLLVM::Intrinsic intrinsicId,bool isOverloadable)987 CWaveBallotIntrinsic(IGCLLVM::Intrinsic intrinsicId, bool isOverloadable)
988 : isaId(GenISAIntrinsic::ID::num_genisa_intrinsics), id(intrinsicId) {}
989
create(GenISAIntrinsic::ID intrinsicId)990 static std::unique_ptr<CWaveBallotIntrinsic> create(GenISAIntrinsic::ID intrinsicId)
991 {
992 return std::unique_ptr<CWaveBallotIntrinsic>(new CWaveBallotIntrinsic(intrinsicId));
993 }
994
createIntrinsic()995 void createIntrinsic()
996 {
997 IGCLLVM::IRBuilder<> IRB(m_pCallInst);
998
999 Value* pSrc = m_pCallInst->getArgOperand(0);
1000 Value* truncInst = IRB.CreateTrunc(pSrc, IRB.getInt1Ty());
1001
1002 if (llvm::Instruction * inst = dyn_cast<llvm::Instruction>(truncInst))
1003 {
1004 inst->setDebugLoc(m_DL);
1005 }
1006
1007 m_args.push_back(truncInst);
1008 replaceGenISACallInst(isaId);
1009 }
1010 };
1011
1012 class CVMESend : public CCommand
1013 {
1014 // id - ID of the intrinsic to be replaced by the call
1015 const GenISAIntrinsic::ID id;
1016
1017 // m_pParamMap - maps image and sampler kernel parameters to BTIs
1018 // and sampler array indexes, respecitvely
1019 CImagesBI::ParamMap* m_pParamMap;
1020
1021 const uint num_images;
1022
1023 public:
CVMESend(GenISAIntrinsic::ID intrinsicId,CImagesBI::ParamMap * paramMap,uint num_images)1024 CVMESend(GenISAIntrinsic::ID intrinsicId, CImagesBI::ParamMap* paramMap, uint num_images)
1025 : id(intrinsicId), m_pParamMap(paramMap), num_images(num_images) {}
1026
create(GenISAIntrinsic::ID intrinsicId,CImagesBI::ParamMap * paramMap,uint num_images=2)1027 static std::unique_ptr<CVMESend> create(GenISAIntrinsic::ID intrinsicId, CImagesBI::ParamMap* paramMap, uint num_images = 2)
1028 {
1029 return std::unique_ptr<CVMESend>(new CVMESend(intrinsicId, paramMap, num_images));
1030 }
1031
createIntrinsic()1032 void createIntrinsic()
1033 {
1034 // First 3 params
1035 m_args.append(m_pCallInst->op_begin(), m_pCallInst->op_begin() + 3);
1036
1037 // Push images like src image, fwd ref image, and bwd ref image.
1038 IGC_ASSERT((num_images >= 2) && (num_images <= 3));
1039 for (uint i = 0; i < num_images; i++) {
1040 Argument* pImg = nullptr;
1041 m_args.push_back(CImagesBI::CImagesUtils::getImageIndex(m_pParamMap, m_pCallInst, i + 3, pImg));
1042 }
1043
1044 // Rest of the params
1045 m_args.append(m_pCallInst->op_begin() + 3 + num_images, m_pCallInst->op_begin() + m_pCallInst->getNumArgOperands());
1046
1047 replaceGenISACallInst(id);
1048 }
1049 };
1050
1051 class CNewVMESend : public CCommand
1052 {
1053 // id - ID of the intrinsic to be replaced by the call
1054 const GenISAIntrinsic::ID id;
1055
1056 // m_pParamMap - maps image and sampler kernel parameters to BTIs
1057 // and sampler array indexes, respectively
1058 CImagesBI::ParamMap* m_pParamMap;
1059
1060 // Number of image arguments
1061 const int m_numImgArgs;
1062 const bool m_IsOverLoaded;
1063
1064 CodeGenContext* m_Ctx;
1065
1066 public:
CNewVMESend(GenISAIntrinsic::ID intrinsicId,CImagesBI::ParamMap * paramMap,CodeGenContext * pCtx,bool isOverloaded,int numImgArgs)1067 CNewVMESend(GenISAIntrinsic::ID intrinsicId, CImagesBI::ParamMap* paramMap, CodeGenContext* pCtx, bool isOverloaded, int numImgArgs)
1068 : id(intrinsicId), m_pParamMap(paramMap), m_numImgArgs(numImgArgs), m_IsOverLoaded(isOverloaded), m_Ctx(pCtx) {}
1069
create(GenISAIntrinsic::ID intrinsicId,CImagesBI::ParamMap * paramMap,CodeGenContext * pCtx,bool isOverloaded,int numImgArgs=3)1070 static std::unique_ptr<CNewVMESend> create(GenISAIntrinsic::ID intrinsicId, CImagesBI::ParamMap* paramMap, CodeGenContext* pCtx, bool isOverloaded, int numImgArgs = 3)
1071 {
1072 return std::unique_ptr<CNewVMESend>(new CNewVMESend(intrinsicId, paramMap, pCtx, isOverloaded, numImgArgs));
1073 }
1074
createIntrinsic()1075 void createIntrinsic()
1076 {
1077 // First 2 params
1078 m_args.append(m_pCallInst->op_begin(), m_pCallInst->op_begin() + 1);
1079
1080 // Push images like src image, fwd ref image, and bwd ref image.
1081 for (int i = 0; i < m_numImgArgs; i++) {
1082 Argument* pImg = nullptr;
1083 m_args.push_back(CImagesBI::CImagesUtils::getImageIndex(m_pParamMap, m_pCallInst, i + 1, pImg));
1084 }
1085
1086 // Rest of the params except for the accelerator sampler.
1087 m_args.append(m_pCallInst->op_begin() + 1 + m_numImgArgs + 1, m_pCallInst->op_begin() + m_pCallInst->getNumArgOperands());
1088
1089 // Device-side VME using the CNewVMESend always use inline samplers.
1090 IGC::ModuleMetaData* MD = m_Ctx->getModuleMetaData();
1091 auto& funcInfo = MD->FuncMD[m_pCallInst->getParent()->getParent()];
1092 funcInfo.hasInlineVmeSamplers = true;
1093
1094 switch (id)
1095 {
1096 case GenISAIntrinsic::GenISA_vmeSendIME2:
1097 {
1098 Type* Tys[] =
1099 {
1100 m_pCallInst->getCalledFunction()->getReturnType(),
1101 m_pCallInst->getArgOperand(0)->getType()
1102 };
1103 replaceGenISACallInst(id, Tys);
1104 break;
1105 }
1106 default:
1107 replaceGenISACallInst(id, m_IsOverLoaded ? ArrayRef<Type*>(m_pCallInst->getArgOperand(0)->getType()) : None);
1108 break;
1109 }
1110 }
1111 };
1112
1113 class CVASend : public CCommand
1114 {
1115 // m_id - ID of the intrinsic to be replaced by the call
1116 const GenISAIntrinsic::ID m_id;
1117
1118 // m_pParamMap - maps image and sampler kernel parameters to BTIs
1119 // and sampler array indexes, respecitvely
1120 CImagesBI::ParamMap* m_pParamMap;
1121
1122 public:
CVASend(GenISAIntrinsic::ID intrinsicId,CImagesBI::ParamMap * paramMap)1123 CVASend(GenISAIntrinsic::ID intrinsicId, CImagesBI::ParamMap* paramMap) : m_id(intrinsicId), m_pParamMap(paramMap) {}
1124
create(GenISAIntrinsic::ID intrinsicId,CImagesBI::ParamMap * paramMap)1125 static std::unique_ptr<CVASend> create(GenISAIntrinsic::ID intrinsicId, CImagesBI::ParamMap* paramMap)
1126 {
1127 return std::unique_ptr<CVASend>(new CVASend(intrinsicId, paramMap));
1128 }
1129
createIntrinsic()1130 void createIntrinsic()
1131 {
1132 m_args.append(m_pCallInst->op_begin(), m_pCallInst->op_end() - 1);
1133
1134 replaceGenISACallInst(m_id);
1135 }
1136 };
1137
1138 class CMemCpy : public CCommand
1139 {
1140 public:
create()1141 static std::unique_ptr<CMemCpy> create()
1142 {
1143 return std::unique_ptr<CMemCpy>(new CMemCpy());
1144 }
1145
createIntrinsic()1146 void createIntrinsic()
1147 {
1148 Value* pDst = m_pCallInst->getArgOperand(0);
1149 Value* pSrc = m_pCallInst->getArgOperand(1);
1150 Value* pNumBytes = m_pCallInst->getArgOperand(2);
1151 uint64_t Align = cast<ConstantInt>(m_pCallInst->getArgOperand(3))->getZExtValue();
1152
1153 IGCLLVM::IRBuilder<> IRB(m_pCallInst);
1154
1155 Value* pDsti8 = IRB.CreateBitCast(
1156 pDst, IRB.getInt8PtrTy(cast<PointerType>(pDst->getType())->getAddressSpace()));
1157
1158 Value* pSrci8 = IRB.CreateBitCast(
1159 pSrc, IRB.getInt8PtrTy(cast<PointerType>(pSrc->getType())->getAddressSpace()));
1160
1161 CallInst* pCall = IRB.CreateMemCpy(pDsti8, pSrci8, pNumBytes, int_cast<unsigned int>(Align));
1162 pCall->setDebugLoc(m_DL);
1163 m_pCallInst->replaceAllUsesWith(pCall);
1164 }
1165 };
1166
1167 class CSamplePos : public CCommand
1168 {
1169 public:
create()1170 static std::unique_ptr<CSamplePos> create()
1171 {
1172 return std::unique_ptr<CSamplePos>(new CSamplePos());
1173 }
1174
createIntrinsic()1175 void createIntrinsic()
1176 {
1177 // WA, platform is not really needed in that case and we cannot access it
1178 PLATFORM platform;
1179 LLVM3DBuilder<> builder(m_pCallInst->getContext(), platform);
1180 builder.SetInsertPoint(m_pCallInst);
1181 Value* textureOperand = m_pCallInst->getOperand(0);
1182 Value* sampleIndex = m_pCallInst->getOperand(1);
1183 Value* samplePosPackedValues = builder.Create_SamplePos(textureOperand, sampleIndex);
1184 cast<llvm::Instruction>(samplePosPackedValues)->setDebugLoc(m_DL);
1185 m_pCallInst->replaceAllUsesWith(samplePosPackedValues);
1186 }
1187 };
1188
1189 template <typename T>
initImageClass(CImagesBI::ParamMap * paramMap,CImagesBI::InlineMap * inlineMap,int * nextSampler,CImagesBI::Dimension dim)1190 std::unique_ptr<T> initImageClass(CImagesBI::ParamMap* paramMap, CImagesBI::InlineMap* inlineMap,
1191 int* nextSampler, CImagesBI::Dimension dim)
1192 {
1193 return std::unique_ptr<T>(new T(paramMap, inlineMap, nextSampler, dim));
1194 }
1195
1196 template <typename T>
initSamplerClass(CImagesBI::ParamMap * paramMap,CImagesBI::InlineMap * inlineMap,int * nextSampler,CImagesBI::Dimension dim,MetaDataUtils * pMdUtils,ModuleMetaData * modMD)1197 std::unique_ptr<T> initSamplerClass(CImagesBI::ParamMap* paramMap, CImagesBI::InlineMap* inlineMap,
1198 int* nextSampler, CImagesBI::Dimension dim, MetaDataUtils* pMdUtils, ModuleMetaData* modMD)
1199 {
1200 return std::unique_ptr<T>(new T(paramMap, inlineMap, nextSampler, dim, pMdUtils, modMD));
1201 }
1202
CBuiltinsResolver(CImagesBI::ParamMap * paramMap,CImagesBI::InlineMap * inlineMap,int * nextSampler,CodeGenContext * ctx)1203 CBuiltinsResolver::CBuiltinsResolver(CImagesBI::ParamMap* paramMap, CImagesBI::InlineMap* inlineMap, int* nextSampler, CodeGenContext* ctx) : m_CodeGenContext(ctx)
1204 {
1205 MetaDataUtils* pMdUtils = ctx->getMetaDataUtils();
1206 ModuleMetaData* modMD = ctx->getModuleMetaData();
1207 // Images Built-ins
1208 // Read_Image builtins
1209 m_CommandMap["__builtin_IB_OCL_1d_ldui"] = initImageClass<COCL_ldui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D);
1210 m_CommandMap["__builtin_IB_OCL_1darr_ldui"] = initImageClass<COCL_ldui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY);
1211 m_CommandMap["__builtin_IB_OCL_2d_ldui"] = initImageClass<COCL_ldui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1212 m_CommandMap["__builtin_IB_OCL_2darr_ldui"] = initImageClass<COCL_ldui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1213 m_CommandMap["__builtin_IB_OCL_3d_ldui"] = initImageClass<COCL_ldui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D);
1214 m_CommandMap["__builtin_IB_OCL_1d_ld"] = initImageClass<COCL_ld>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D);
1215 m_CommandMap["__builtin_IB_OCL_1darr_ld"] = initImageClass<COCL_ld>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY);
1216 m_CommandMap["__builtin_IB_OCL_2d_ld"] = initImageClass<COCL_ld>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1217 m_CommandMap["__builtin_IB_OCL_2darr_ld"] = initImageClass<COCL_ld>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1218 m_CommandMap["__builtin_IB_OCL_3d_ld"] = initImageClass<COCL_ld>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D);
1219 m_CommandMap["__builtin_IB_OCL_2d_ldmcs"] = initImageClass<COCL_ldmcs>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1220 m_CommandMap["__builtin_IB_OCL_2darr_ldmcs"] = initImageClass<COCL_ldmcs>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1221 m_CommandMap["__builtin_IB_OCL_2d_ld2dms"] = initImageClass<COCL_ld2dms>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1222 m_CommandMap["__builtin_IB_OCL_2darr_ld2dms"] = initImageClass<COCL_ld2dms>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1223 m_CommandMap["__builtin_IB_OCL_2d_ld2dmsui"] = initImageClass<COCL_ld2dmsui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1224 m_CommandMap["__builtin_IB_OCL_2darr_ld2dmsui"] = initImageClass<COCL_ld2dmsui>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1225 m_CommandMap["__builtin_IB_OCL_1d_sample_l"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D, pMdUtils, modMD);
1226 m_CommandMap["__builtin_IB_OCL_1darr_sample_l"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY, pMdUtils, modMD);
1227 m_CommandMap["__builtin_IB_OCL_2d_sample_l"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D, pMdUtils, modMD);
1228 m_CommandMap["__builtin_IB_OCL_2darr_sample_l"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY, pMdUtils, modMD);
1229 m_CommandMap["__builtin_IB_OCL_3d_sample_l"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D, pMdUtils, modMD);
1230 m_CommandMap["__builtin_IB_OCL_1d_sample_d"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D, pMdUtils, modMD);
1231 m_CommandMap["__builtin_IB_OCL_1darr_sample_d"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY, pMdUtils, modMD);
1232 m_CommandMap["__builtin_IB_OCL_2d_sample_d"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D, pMdUtils, modMD);
1233 m_CommandMap["__builtin_IB_OCL_2darr_sample_d"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY, pMdUtils, modMD);
1234 m_CommandMap["__builtin_IB_OCL_3d_sample_d"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D, pMdUtils, modMD);
1235 m_CommandMap["__builtin_IB_OCL_1d_sample_lui"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D, pMdUtils, modMD);
1236 m_CommandMap["__builtin_IB_OCL_1darr_sample_lui"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY, pMdUtils, modMD);
1237 m_CommandMap["__builtin_IB_OCL_2d_sample_lui"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D, pMdUtils, modMD);
1238 m_CommandMap["__builtin_IB_OCL_2darr_sample_lui"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY, pMdUtils, modMD);
1239 m_CommandMap["__builtin_IB_OCL_3d_sample_lui"] = initSamplerClass<COCL_sample_l>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D, pMdUtils, modMD);
1240 m_CommandMap["__builtin_IB_OCL_1d_sample_dui"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D, pMdUtils, modMD);
1241 m_CommandMap["__builtin_IB_OCL_1darr_sample_dui"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY, pMdUtils, modMD);
1242 m_CommandMap["__builtin_IB_OCL_2d_sample_dui"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D, pMdUtils, modMD);
1243 m_CommandMap["__builtin_IB_OCL_2darr_sample_dui"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY, pMdUtils, modMD);
1244 m_CommandMap["__builtin_IB_OCL_3d_sample_dui"] = initSamplerClass<COCL_sample_d>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D, pMdUtils, modMD);
1245
1246 // Write Image
1247 m_CommandMap["__builtin_IB_write_1d_ui"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D);
1248 m_CommandMap["__builtin_IB_write_1darr_ui"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_1D_ARRAY);
1249 m_CommandMap["__builtin_IB_write_2d_ui"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1250 m_CommandMap["__builtin_IB_write_2darr_ui"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1251 m_CommandMap["__builtin_IB_write_3d_ui"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_3D);
1252 m_CommandMap["__builtin_IB_write_2d_f"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D);
1253 m_CommandMap["__builtin_IB_write_2darr_f"] = initImageClass<CWrite>(paramMap, inlineMap, nextSampler, CImagesBI::Dimension::DIM_2D_ARRAY);
1254
1255 //convert Built-ins
1256 m_CommandMap["__builtin_IB_frnd_ne"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ROUNDNE, false);
1257 m_CommandMap["__builtin_IB_ftoh_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtn, true);
1258 m_CommandMap["__builtin_IB_ftoh_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtp, true);
1259 m_CommandMap["__builtin_IB_ftoh_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtz, true);
1260 m_CommandMap["__builtin_IB_dtoh_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtn, true);
1261 m_CommandMap["__builtin_IB_dtoh_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtp, true);
1262 m_CommandMap["__builtin_IB_dtoh_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtz, true);
1263 m_CommandMap["__builtin_IB_dtof_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtn, true);
1264 m_CommandMap["__builtin_IB_dtof_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtp, true);
1265 m_CommandMap["__builtin_IB_dtof_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftof_rtz, true);
1266
1267 m_CommandMap["__builtin_IB_dtoi8_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtn, true);
1268 m_CommandMap["__builtin_IB_dtoi8_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtp, true);
1269 m_CommandMap["__builtin_IB_dtoi8_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rte, true);
1270 m_CommandMap["__builtin_IB_dtoi16_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtn, true);
1271 m_CommandMap["__builtin_IB_dtoi16_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtp, true);
1272 m_CommandMap["__builtin_IB_dtoi16_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rte, true);
1273 m_CommandMap["__builtin_IB_dtoi32_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtn, true);
1274 m_CommandMap["__builtin_IB_dtoi32_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtp, true);
1275 m_CommandMap["__builtin_IB_dtoi32_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rte, true);
1276 m_CommandMap["__builtin_IB_dtoi64_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtn, true);
1277 m_CommandMap["__builtin_IB_dtoi64_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rtp, true);
1278 m_CommandMap["__builtin_IB_dtoi64_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoi_rte, true);
1279
1280 m_CommandMap["__builtin_IB_dtoui8_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtn, true);
1281 m_CommandMap["__builtin_IB_dtoui8_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtp, true);
1282 m_CommandMap["__builtin_IB_dtoui8_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rte, true);
1283 m_CommandMap["__builtin_IB_dtoui16_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtn, true);
1284 m_CommandMap["__builtin_IB_dtoui16_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtp, true);
1285 m_CommandMap["__builtin_IB_dtoui16_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rte, true);
1286 m_CommandMap["__builtin_IB_dtoui32_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtn, true);
1287 m_CommandMap["__builtin_IB_dtoui32_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtp, true);
1288 m_CommandMap["__builtin_IB_dtoui32_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rte, true);
1289 m_CommandMap["__builtin_IB_dtoui64_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtn, true);
1290 m_CommandMap["__builtin_IB_dtoui64_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rtp, true);
1291 m_CommandMap["__builtin_IB_dtoui64_rte"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ftoui_rte, true);
1292
1293 m_CommandMap["__builtin_IB_itof_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_itof_rtn, true);
1294 m_CommandMap["__builtin_IB_itof_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_itof_rtp, true);
1295 m_CommandMap["__builtin_IB_itof_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_itof_rtz, true);
1296 m_CommandMap["__builtin_IB_uitof_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_uitof_rtn, true);
1297 m_CommandMap["__builtin_IB_uitof_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_uitof_rtp, true);
1298 m_CommandMap["__builtin_IB_uitof_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_uitof_rtz, true);
1299
1300 m_CommandMap["__builtin_IB_itofp64_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_itof_rtn, true);
1301 m_CommandMap["__builtin_IB_itofp64_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_itof_rtp, true);
1302 m_CommandMap["__builtin_IB_itofp64_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_itof_rtz, true);
1303 m_CommandMap["__builtin_IB_uitofp64_rtn"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_uitof_rtn, true);
1304 m_CommandMap["__builtin_IB_uitofp64_rtp"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_uitof_rtp, true);
1305 m_CommandMap["__builtin_IB_uitofp64_rtz"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_uitof_rtz, true);
1306
1307 //Internal Debug Built-Ins
1308 m_CommandMap["__builtin_IB_read_cycle_counter"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_cycleCounter, false);
1309 m_CommandMap["__builtin_IB_source_value"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_source_value, false);
1310 m_CommandMap["__builtin_IB_set_dbg_register"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_SetDebugReg, false);
1311 m_CommandMap["__builtin_IB_movflag"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_movflag, false);
1312 m_CommandMap["__builtin_IB_movcr"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_movcr, false);
1313 m_CommandMap["__builtin_IB_hw_thread_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_hw_thread_id, false);
1314 m_CommandMap["__builtin_IB_slice_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_slice_id, false);
1315 m_CommandMap["__builtin_IB_subslice_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_subslice_id, false);
1316 m_CommandMap["__builtin_IB_dual_subslice_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_dual_subslice_id, false);
1317 m_CommandMap["__builtin_IB_eu_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_eu_id, false);
1318 m_CommandMap["__builtin_IB_get_sr0"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getSR0, false);
1319 m_CommandMap["__builtin_IB_eu_thread_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_eu_thread_id, false);
1320 m_CommandMap["__builtin_IB_eu_thread_pause"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_eu_thread_pause, false);
1321
1322 //math Built-ins
1323 m_CommandMap["__builtin_IB_frnd_pi"] = CSimpleIntrinMapping::create(Intrinsic::ceil);
1324 m_CommandMap["__builtin_IB_frnd_ni"] = CSimpleIntrinMapping::create(Intrinsic::floor);
1325 m_CommandMap["__builtin_IB_frnd_zi"] = CSimpleIntrinMapping::create(Intrinsic::trunc);
1326 m_CommandMap["__builtin_IB_native_cosf"] = CSimpleIntrinMapping::create(Intrinsic::cos);
1327 m_CommandMap["__builtin_IB_native_cosh"] = CSimpleIntrinMapping::create(Intrinsic::cos);
1328 m_CommandMap["__builtin_IB_native_sinf"] = CSimpleIntrinMapping::create(Intrinsic::sin);
1329 m_CommandMap["__builtin_IB_native_sinh"] = CSimpleIntrinMapping::create(Intrinsic::sin);
1330 m_CommandMap["__builtin_IB_native_exp2f"] = CSimpleIntrinMapping::create(Intrinsic::exp2);
1331 m_CommandMap["__builtin_IB_native_exp2h"] = CSimpleIntrinMapping::create(Intrinsic::exp2);
1332 m_CommandMap["__builtin_IB_native_log2f"] = CSimpleIntrinMapping::create(Intrinsic::log2);
1333 m_CommandMap["__builtin_IB_native_log2h"] = CSimpleIntrinMapping::create(Intrinsic::log2);
1334 m_CommandMap["__builtin_IB_native_sqrtf"] = CSimpleIntrinMapping::create(Intrinsic::sqrt);
1335 m_CommandMap["__builtin_IB_native_sqrth"] = CSimpleIntrinMapping::create(Intrinsic::sqrt);
1336 m_CommandMap["__builtin_IB_native_sqrtd"] = CSimpleIntrinMapping::create(Intrinsic::sqrt);
1337 m_CommandMap["__builtin_IB_popcount_1u32"] = CSimpleIntrinMapping::create(Intrinsic::ctpop);
1338 m_CommandMap["__builtin_IB_popcount_1u16"] = CSimpleIntrinMapping::create(Intrinsic::ctpop);
1339 m_CommandMap["__builtin_IB_popcount_1u8"] = CSimpleIntrinMapping::create(Intrinsic::ctpop);
1340 m_CommandMap["__builtin_IB_native_powrf"] = CSimpleIntrinMapping::create(Intrinsic::pow);
1341 m_CommandMap["__builtin_IB_fma"] = CSimpleIntrinMapping::create(Intrinsic::fma);
1342 m_CommandMap["__builtin_IB_fmah"] = CSimpleIntrinMapping::create(Intrinsic::fma);
1343 m_CommandMap["__builtin_IB_bfi"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_bfi, false);
1344 m_CommandMap["__builtin_IB_ibfe"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ibfe, false);
1345 m_CommandMap["__builtin_IB_ubfe"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_ubfe, false);
1346 m_CommandMap["__builtin_IB_bfrev"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_bfrev, false);
1347 m_CommandMap["__builtin_IB_fmax"] = CSimpleIntrinMapping::create(Intrinsic::maxnum);
1348 m_CommandMap["__builtin_IB_fmin"] = CSimpleIntrinMapping::create(Intrinsic::minnum);
1349 m_CommandMap["__builtin_IB_HMAX"] = CSimpleIntrinMapping::create(Intrinsic::maxnum);
1350 m_CommandMap["__builtin_IB_HMIN"] = CSimpleIntrinMapping::create(Intrinsic::minnum);
1351 m_CommandMap["__builtin_IB_dmin"] = CSimpleIntrinMapping::create(Intrinsic::minnum);
1352 m_CommandMap["__builtin_IB_dmax"] = CSimpleIntrinMapping::create(Intrinsic::maxnum);
1353 m_CommandMap["__builtin_IB_mul_rtz_f64"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_mul_rtz);
1354 m_CommandMap["__builtin_IB_mul_rtz_f32"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_mul_rtz);
1355 m_CommandMap["__builtin_IB_fma_rtz_f64"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_fma_rtz);
1356 m_CommandMap["__builtin_IB_fma_rtz_f32"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_fma_rtz);
1357 m_CommandMap["__builtin_IB_add_rtz_f64"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_add_rtz);
1358 m_CommandMap["__builtin_IB_add_rtz_f32"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_add_rtz);
1359 m_CommandMap["__builtin_IB_fma_rtp_f64"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_fma_rtp);
1360 m_CommandMap["__builtin_IB_fma_rtn_f64"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_fma_rtn);
1361
1362 //Sync built-ins
1363 m_CommandMap["__builtin_IB_thread_group_barrier"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_threadgroupbarrier, false);
1364 m_CommandMap["__builtin_IB_thread_group_barrier_signal"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_threadgroupbarrier_signal, false);
1365 m_CommandMap["__builtin_IB_thread_group_barrier_wait"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_threadgroupbarrier_wait, false);
1366 m_CommandMap["__builtin_IB_memfence"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_memoryfence, false);
1367 m_CommandMap["__builtin_IB_flush_sampler_cache"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_flushsampler, false);
1368 m_CommandMap["__builtin_IB_typedmemfence"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_typedmemoryfence, false);
1369 // internal hint builtin
1370 m_CommandMap["__builtin_IB_assume_uniform"] = CSimpleIntrinMapping::create( GenISAIntrinsic::GenISA_assume_uniform, true );
1371
1372 // helper built-ins
1373 m_CommandMap["__builtin_IB_simd_lane_id"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdLaneId, false);
1374
1375 // vme builtins
1376 m_CommandMap["__builtin_IB_vme_send_ime"] = CVMESend::create(GenISAIntrinsic::GenISA_vmeSendIME, paramMap);
1377 m_CommandMap["__builtin_IB_vme_send_fbr"] = CVMESend::create(GenISAIntrinsic::GenISA_vmeSendFBR, paramMap);
1378 m_CommandMap["__builtin_IB_vme_send_sic"] = CVMESend::create(GenISAIntrinsic::GenISA_vmeSendSIC, paramMap, 3);
1379
1380 m_CommandMap["__builtin_IB_vme_send_fbr_new"] = CNewVMESend::create(GenISAIntrinsic::GenISA_vmeSendFBR2, paramMap, ctx, false);
1381 m_CommandMap["__builtin_IB_vme_send_sic_new"] = CNewVMESend::create(GenISAIntrinsic::GenISA_vmeSendSIC2, paramMap, ctx, false);
1382 m_CommandMap["__builtin_IB_vme_send_ime_new_uint4_uint8"] = CNewVMESend::create(GenISAIntrinsic::GenISA_vmeSendIME2, paramMap, ctx, true);
1383 m_CommandMap["__builtin_IB_vme_send_ime_new_uint8_uint8"] = CNewVMESend::create(GenISAIntrinsic::GenISA_vmeSendIME2, paramMap, ctx, true);
1384 m_CommandMap["__builtin_IB_vme_send_ime_new_uint4_uint4"] = CNewVMESend::create(GenISAIntrinsic::GenISA_vmeSendIME2, paramMap, ctx, true);
1385 m_CommandMap["__builtin_IB_vme_send_ime_new_uint8_uint4"] = CNewVMESend::create(GenISAIntrinsic::GenISA_vmeSendIME2, paramMap, ctx, true);
1386
1387 m_CommandMap["__builtin_IB_set_message_phase_legacy_dw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseX_legacy, true);
1388 m_CommandMap["__builtin_IB_set_message_phase_legacy_uw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseX_legacy, true);
1389 m_CommandMap["__builtin_IB_set_message_phase_legacy_ub"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseX_legacy, true);
1390
1391 m_CommandMap["__builtin_IB_set_message_phase_legacy"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhase_legacy, false);
1392
1393 m_CommandMap["__builtin_IB_create_message_phases"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhases, false);
1394 m_CommandMap["__builtin_IB_create_message_phases_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesV, true);
1395 m_CommandMap["__builtin_IB_create_message_phases_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesV, true);
1396 m_CommandMap["__builtin_IB_create_message_phases_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesV, true);
1397
1398 m_CommandMap["__builtin_IB_create_message_phases_no_init"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesNoInit, false);
1399 m_CommandMap["__builtin_IB_create_message_phases_no_init_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesNoInitV, true);
1400 m_CommandMap["__builtin_IB_create_message_phases_no_init_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesNoInitV, true);
1401 m_CommandMap["__builtin_IB_create_message_phases_no_init_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_createMessagePhasesNoInitV, true);
1402
1403 m_CommandMap["__builtin_IB_get_message_phase_dw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseX, true);
1404 m_CommandMap["__builtin_IB_get_message_phase_dw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1405 m_CommandMap["__builtin_IB_get_message_phase_dw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1406 m_CommandMap["__builtin_IB_get_message_phase_dw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1407
1408 m_CommandMap["__builtin_IB_get_message_phase_uq"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseX, true);
1409 m_CommandMap["__builtin_IB_get_message_phase_uq_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1410 m_CommandMap["__builtin_IB_get_message_phase_uq_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1411 m_CommandMap["__builtin_IB_get_message_phase_uq_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1412
1413 m_CommandMap["__builtin_IB_set_message_phase_dw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseX, true);
1414 m_CommandMap["__builtin_IB_set_message_phase_dw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1415 m_CommandMap["__builtin_IB_set_message_phase_dw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1416 m_CommandMap["__builtin_IB_set_message_phase_dw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1417
1418 m_CommandMap["__builtin_IB_get_message_phase_uw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseX, true);
1419 m_CommandMap["__builtin_IB_get_message_phase_uw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1420 m_CommandMap["__builtin_IB_get_message_phase_uw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1421 m_CommandMap["__builtin_IB_get_message_phase_uw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1422
1423 m_CommandMap["__builtin_IB_set_message_phase_uw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseX, true);
1424 m_CommandMap["__builtin_IB_set_message_phase_uw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1425 m_CommandMap["__builtin_IB_set_message_phase_uw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1426 m_CommandMap["__builtin_IB_set_message_phase_uw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1427
1428 m_CommandMap["__builtin_IB_get_message_phase_ub"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseX, true);
1429 m_CommandMap["__builtin_IB_get_message_phase_ub_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1430 m_CommandMap["__builtin_IB_get_message_phase_ub_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1431 m_CommandMap["__builtin_IB_get_message_phase_ub_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseXV, true);
1432
1433 m_CommandMap["__builtin_IB_set_message_phase_ub"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseX, true);
1434 m_CommandMap["__builtin_IB_set_message_phase_ub_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1435 m_CommandMap["__builtin_IB_set_message_phase_ub_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1436 m_CommandMap["__builtin_IB_set_message_phase_ub_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseXV, true);
1437
1438 m_CommandMap["__builtin_IB_get_message_phase"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhase, false);
1439 m_CommandMap["__builtin_IB_get_message_phase_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseV, true);
1440 m_CommandMap["__builtin_IB_get_message_phase_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseV, true);
1441 m_CommandMap["__builtin_IB_get_message_phase_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_getMessagePhaseV, true);
1442
1443 m_CommandMap["__builtin_IB_set_message_phase"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhase, false);
1444 m_CommandMap["__builtin_IB_set_message_phase_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseV, true);
1445 m_CommandMap["__builtin_IB_set_message_phase_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseV, true);
1446 m_CommandMap["__builtin_IB_set_message_phase_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_setMessagePhaseV, true);
1447
1448 m_CommandMap["__builtin_IB_broadcast_message_phase_ub"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhase, true);
1449 m_CommandMap["__builtin_IB_broadcast_message_phase_ub_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1450 m_CommandMap["__builtin_IB_broadcast_message_phase_ub_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1451 m_CommandMap["__builtin_IB_broadcast_message_phase_ub_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1452
1453 m_CommandMap["__builtin_IB_broadcast_message_phase_uw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhase, true);
1454 m_CommandMap["__builtin_IB_broadcast_message_phase_uw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1455 m_CommandMap["__builtin_IB_broadcast_message_phase_uw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1456 m_CommandMap["__builtin_IB_broadcast_message_phase_uw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1457
1458 m_CommandMap["__builtin_IB_broadcast_message_phase_dw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhase, true);
1459 m_CommandMap["__builtin_IB_broadcast_message_phase_dw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1460 m_CommandMap["__builtin_IB_broadcast_message_phase_dw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1461 m_CommandMap["__builtin_IB_broadcast_message_phase_dw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1462
1463 m_CommandMap["__builtin_IB_broadcast_message_phase_uq"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhase, true);
1464 m_CommandMap["__builtin_IB_broadcast_message_phase_uq_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1465 m_CommandMap["__builtin_IB_broadcast_message_phase_uq_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1466 m_CommandMap["__builtin_IB_broadcast_message_phase_uq_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_broadcastMessagePhaseV, true);
1467
1468 m_CommandMap["__builtin_IB_simd_set_message_phase_ub"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhase, true);
1469 m_CommandMap["__builtin_IB_simd_set_message_phase_ub_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1470 m_CommandMap["__builtin_IB_simd_set_message_phase_ub_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1471 m_CommandMap["__builtin_IB_simd_set_message_phase_ub_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1472
1473 m_CommandMap["__builtin_IB_simd_set_message_phase_uw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhase, true);
1474 m_CommandMap["__builtin_IB_simd_set_message_phase_uw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1475 m_CommandMap["__builtin_IB_simd_set_message_phase_uw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1476 m_CommandMap["__builtin_IB_simd_set_message_phase_uw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1477
1478 m_CommandMap["__builtin_IB_simd_set_message_phase_dw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhase, true);
1479 m_CommandMap["__builtin_IB_simd_set_message_phase_dw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1480 m_CommandMap["__builtin_IB_simd_set_message_phase_dw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1481 m_CommandMap["__builtin_IB_simd_set_message_phase_dw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1482
1483 m_CommandMap["__builtin_IB_simd_set_message_phase_uq"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhase, true);
1484 m_CommandMap["__builtin_IB_simd_set_message_phase_uq_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1485 m_CommandMap["__builtin_IB_simd_set_message_phase_uq_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1486 m_CommandMap["__builtin_IB_simd_set_message_phase_uq_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdSetMessagePhaseV, true);
1487
1488 m_CommandMap["__builtin_IB_simd_get_message_phase_uw"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhase, true);
1489 m_CommandMap["__builtin_IB_simd_get_message_phase_uw_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhaseV, true);
1490 m_CommandMap["__builtin_IB_simd_get_message_phase_uw_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhaseV, true);
1491 m_CommandMap["__builtin_IB_simd_get_message_phase_uw_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhaseV, true);
1492
1493 m_CommandMap["__builtin_IB_simd_get_message_phase_uq"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhase, true);
1494 m_CommandMap["__builtin_IB_simd_get_message_phase_uq_uint2"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhaseV, true);
1495 m_CommandMap["__builtin_IB_simd_get_message_phase_uq_uint4"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhaseV, true);
1496 m_CommandMap["__builtin_IB_simd_get_message_phase_uq_uint8"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdGetMessagePhaseV, true);
1497
1498 m_CommandMap["__builtin_IB_extract_mv_and_sad"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_extractMVAndSAD, false);
1499 m_CommandMap["__builtin_IB_cmp_sads"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_cmpSADs, false);
1500
1501 m_CommandMap["__builtin_IB_simdMediaRegionCopy"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_simdMediaRegionCopy, false);
1502
1503 // builtin to emit a memcpy intrinsic
1504 m_CommandMap["__builtin_IB_memcpy_global_to_private"] = CMemCpy::create();
1505 m_CommandMap["__builtin_IB_memcpy_constant_to_private"] = CMemCpy::create();
1506 m_CommandMap["__builtin_IB_memcpy_local_to_private"] = CMemCpy::create();
1507 m_CommandMap["__builtin_IB_memcpy_private_to_private"] = CMemCpy::create();
1508 m_CommandMap["__builtin_IB_memcpy_generic_to_private"] = CMemCpy::create();
1509 m_CommandMap["__builtin_IB_memcpy_private_to_global"] = CMemCpy::create();
1510 m_CommandMap["__builtin_IB_memcpy_private_to_constant"] = CMemCpy::create();
1511 m_CommandMap["__builtin_IB_memcpy_private_to_local"] = CMemCpy::create();
1512 m_CommandMap["__builtin_IB_memcpy_private_to_generic"] = CMemCpy::create();
1513
1514 // Correctly Rounded built ins
1515 m_CommandMap["__builtin_IB_ieee_sqrt"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_IEEE_Sqrt, false);
1516 m_CommandMap["__builtin_IB_ieee_divide"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_IEEE_Divide, true);
1517 m_CommandMap["__builtin_IB_ieee_divide_f64"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_IEEE_Divide, true);
1518
1519 // VA built-ins
1520 m_CommandMap["__builtin_IB_va_erode_64x4"] = CVASend::create(GenISAIntrinsic::GenISA_vaErode, paramMap);
1521 m_CommandMap["__builtin_IB_va_dilate_64x4"] = CVASend::create(GenISAIntrinsic::GenISA_vaDilate, paramMap);
1522 m_CommandMap["__builtin_IB_va_minmaxfilter_16x4_SLM"] = CVASend::create(GenISAIntrinsic::GenISA_vaMinMaxFilter, paramMap);
1523 m_CommandMap["__builtin_IB_va_convolve_16x4_SLM"] = CVASend::create(GenISAIntrinsic::GenISA_vaConvolve, paramMap);
1524 m_CommandMap["__builtin_IB_va_convolve_16x1"] = CVASend::create(GenISAIntrinsic::GenISA_vaConvolveGRF_16x1, paramMap);
1525 m_CommandMap["__builtin_IB_va_convolve_16x4"] = CVASend::create(GenISAIntrinsic::GenISA_vaConvolveGRF_16x4, paramMap);
1526 m_CommandMap["__builtin_IB_va_minmax"] = CVASend::create(GenISAIntrinsic::GenISA_vaMinMax, paramMap);
1527 m_CommandMap["__builtin_IB_va_centroid"] = CVASend::create(GenISAIntrinsic::GenISA_vaCentroid, paramMap);
1528 m_CommandMap["__builtin_IB_va_boolcentroid"] = CVASend::create(GenISAIntrinsic::GenISA_vaBoolCentroid, paramMap);
1529 m_CommandMap["__builtin_IB_va_boolsum"] = CVASend::create(GenISAIntrinsic::GenISA_vaBoolSum, paramMap);
1530
1531 // Ballot builtins
1532 m_CommandMap["__builtin_IB_WaveBallot"] = CWaveBallotIntrinsic::create(GenISAIntrinsic::GenISA_WaveBallot);
1533
1534 m_CommandMap[StringRef("__builtin_IB_samplepos")] = CSamplePos::create();
1535
1536 // `dp4a` built-ins
1537 m_CommandMap["__builtin_IB_dp4a_ss"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_dp4a_ss, false);
1538 m_CommandMap["__builtin_IB_dp4a_uu"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_dp4a_uu, false);
1539 m_CommandMap["__builtin_IB_dp4a_su"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_dp4a_su, false);
1540 m_CommandMap["__builtin_IB_dp4a_us"] = CSimpleIntrinMapping::create(GenISAIntrinsic::GenISA_dp4a_us, false);
1541
1542 m_KnownBuiltins = {
1543 //resolved later in GenericAddressDynamicResolution pass:
1544 "__builtin_IB_to_local",
1545 "__builtin_IB_to_private"
1546 };
1547 }
1548
resolveBI(CallInst * Inst)1549 bool CBuiltinsResolver::resolveBI(CallInst* Inst)
1550 {
1551 Function* callee = Inst->getCalledFunction();
1552 StringRef calleeName = callee->getName();
1553 //Check if it exists in the map.
1554 if (!m_CommandMap.count(calleeName))
1555 {
1556 return false;
1557 }
1558 m_CommandMap[calleeName]->execute(Inst, m_CodeGenContext);
1559 m_CommandMap[calleeName]->verifyCommand();
1560
1561 return !m_CodeGenContext->HasError();
1562 }
1563