1 //===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains miscellaneous utility functions
11 //===----------------------------------------------------------------------===//
12
13 #include "NVPTXUtilities.h"
14 #include "NVPTX.h"
15 #include "llvm/IR/Constants.h"
16 #include "llvm/IR/Function.h"
17 #include "llvm/IR/GlobalVariable.h"
18 #include "llvm/IR/InstIterator.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/IR/Operator.h"
21 #include "llvm/Support/ManagedStatic.h"
22 #include "llvm/Support/MutexGuard.h"
23 #include <algorithm>
24 #include <cstring>
25 #include <map>
26 #include <string>
27 #include <vector>
28
29 using namespace llvm;
30
31 typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t;
32 typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;
33 typedef std::map<const Module *, global_val_annot_t> per_module_annot_t;
34
35 ManagedStatic<per_module_annot_t> annotationCache;
36 static sys::Mutex Lock;
37
clearAnnotationCache(const llvm::Module * Mod)38 void llvm::clearAnnotationCache(const llvm::Module *Mod) {
39 MutexGuard Guard(Lock);
40 annotationCache->erase(Mod);
41 }
42
cacheAnnotationFromMD(const MDNode * md,key_val_pair_t & retval)43 static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval) {
44 MutexGuard Guard(Lock);
45 assert(md && "Invalid mdnode for annotation");
46 assert((md->getNumOperands() % 2) == 1 && "Invalid number of operands");
47 // start index = 1, to skip the global variable key
48 // increment = 2, to skip the value for each property-value pairs
49 for (unsigned i = 1, e = md->getNumOperands(); i != e; i += 2) {
50 // property
51 const MDString *prop = dyn_cast<MDString>(md->getOperand(i));
52 assert(prop && "Annotation property not a string");
53
54 // value
55 ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(md->getOperand(i + 1));
56 assert(Val && "Value operand not a constant int");
57
58 std::string keyname = prop->getString().str();
59 if (retval.find(keyname) != retval.end())
60 retval[keyname].push_back(Val->getZExtValue());
61 else {
62 std::vector<unsigned> tmp;
63 tmp.push_back(Val->getZExtValue());
64 retval[keyname] = tmp;
65 }
66 }
67 }
68
cacheAnnotationFromMD(const Module * m,const GlobalValue * gv)69 static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
70 MutexGuard Guard(Lock);
71 NamedMDNode *NMD = m->getNamedMetadata(llvm::NamedMDForAnnotations);
72 if (!NMD)
73 return;
74 key_val_pair_t tmp;
75 for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
76 const MDNode *elem = NMD->getOperand(i);
77
78 GlobalValue *entity =
79 mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0));
80 // entity may be null due to DCE
81 if (!entity)
82 continue;
83 if (entity != gv)
84 continue;
85
86 // accumulate annotations for entity in tmp
87 cacheAnnotationFromMD(elem, tmp);
88 }
89
90 if (tmp.empty()) // no annotations for this gv
91 return;
92
93 if ((*annotationCache).find(m) != (*annotationCache).end())
94 (*annotationCache)[m][gv] = std::move(tmp);
95 else {
96 global_val_annot_t tmp1;
97 tmp1[gv] = std::move(tmp);
98 (*annotationCache)[m] = std::move(tmp1);
99 }
100 }
101
findOneNVVMAnnotation(const GlobalValue * gv,std::string prop,unsigned & retval)102 bool llvm::findOneNVVMAnnotation(const GlobalValue *gv, std::string prop,
103 unsigned &retval) {
104 MutexGuard Guard(Lock);
105 const Module *m = gv->getParent();
106 if ((*annotationCache).find(m) == (*annotationCache).end())
107 cacheAnnotationFromMD(m, gv);
108 else if ((*annotationCache)[m].find(gv) == (*annotationCache)[m].end())
109 cacheAnnotationFromMD(m, gv);
110 if ((*annotationCache)[m][gv].find(prop) == (*annotationCache)[m][gv].end())
111 return false;
112 retval = (*annotationCache)[m][gv][prop][0];
113 return true;
114 }
115
findAllNVVMAnnotation(const GlobalValue * gv,std::string prop,std::vector<unsigned> & retval)116 bool llvm::findAllNVVMAnnotation(const GlobalValue *gv, std::string prop,
117 std::vector<unsigned> &retval) {
118 MutexGuard Guard(Lock);
119 const Module *m = gv->getParent();
120 if ((*annotationCache).find(m) == (*annotationCache).end())
121 cacheAnnotationFromMD(m, gv);
122 else if ((*annotationCache)[m].find(gv) == (*annotationCache)[m].end())
123 cacheAnnotationFromMD(m, gv);
124 if ((*annotationCache)[m][gv].find(prop) == (*annotationCache)[m][gv].end())
125 return false;
126 retval = (*annotationCache)[m][gv][prop];
127 return true;
128 }
129
isTexture(const llvm::Value & val)130 bool llvm::isTexture(const llvm::Value &val) {
131 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
132 unsigned annot;
133 if (llvm::findOneNVVMAnnotation(
134 gv, llvm::PropertyAnnotationNames[llvm::PROPERTY_ISTEXTURE],
135 annot)) {
136 assert((annot == 1) && "Unexpected annotation on a texture symbol");
137 return true;
138 }
139 }
140 return false;
141 }
142
isSurface(const llvm::Value & val)143 bool llvm::isSurface(const llvm::Value &val) {
144 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
145 unsigned annot;
146 if (llvm::findOneNVVMAnnotation(
147 gv, llvm::PropertyAnnotationNames[llvm::PROPERTY_ISSURFACE],
148 annot)) {
149 assert((annot == 1) && "Unexpected annotation on a surface symbol");
150 return true;
151 }
152 }
153 return false;
154 }
155
isSampler(const llvm::Value & val)156 bool llvm::isSampler(const llvm::Value &val) {
157 if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
158 unsigned annot;
159 if (llvm::findOneNVVMAnnotation(
160 gv, llvm::PropertyAnnotationNames[llvm::PROPERTY_ISSAMPLER],
161 annot)) {
162 assert((annot == 1) && "Unexpected annotation on a sampler symbol");
163 return true;
164 }
165 }
166 if (const Argument *arg = dyn_cast<Argument>(&val)) {
167 const Function *func = arg->getParent();
168 std::vector<unsigned> annot;
169 if (llvm::findAllNVVMAnnotation(
170 func, llvm::PropertyAnnotationNames[llvm::PROPERTY_ISSAMPLER],
171 annot)) {
172 if (std::find(annot.begin(), annot.end(), arg->getArgNo()) != annot.end())
173 return true;
174 }
175 }
176 return false;
177 }
178
isImageReadOnly(const llvm::Value & val)179 bool llvm::isImageReadOnly(const llvm::Value &val) {
180 if (const Argument *arg = dyn_cast<Argument>(&val)) {
181 const Function *func = arg->getParent();
182 std::vector<unsigned> annot;
183 if (llvm::findAllNVVMAnnotation(func,
184 llvm::PropertyAnnotationNames[
185 llvm::PROPERTY_ISREADONLY_IMAGE_PARAM],
186 annot)) {
187 if (std::find(annot.begin(), annot.end(), arg->getArgNo()) != annot.end())
188 return true;
189 }
190 }
191 return false;
192 }
193
isImageWriteOnly(const llvm::Value & val)194 bool llvm::isImageWriteOnly(const llvm::Value &val) {
195 if (const Argument *arg = dyn_cast<Argument>(&val)) {
196 const Function *func = arg->getParent();
197 std::vector<unsigned> annot;
198 if (llvm::findAllNVVMAnnotation(func,
199 llvm::PropertyAnnotationNames[
200 llvm::PROPERTY_ISWRITEONLY_IMAGE_PARAM],
201 annot)) {
202 if (std::find(annot.begin(), annot.end(), arg->getArgNo()) != annot.end())
203 return true;
204 }
205 }
206 return false;
207 }
208
isImageReadWrite(const llvm::Value & val)209 bool llvm::isImageReadWrite(const llvm::Value &val) {
210 if (const Argument *arg = dyn_cast<Argument>(&val)) {
211 const Function *func = arg->getParent();
212 std::vector<unsigned> annot;
213 if (llvm::findAllNVVMAnnotation(func,
214 llvm::PropertyAnnotationNames[
215 llvm::PROPERTY_ISREADWRITE_IMAGE_PARAM],
216 annot)) {
217 if (std::find(annot.begin(), annot.end(), arg->getArgNo()) != annot.end())
218 return true;
219 }
220 }
221 return false;
222 }
223
isImage(const llvm::Value & val)224 bool llvm::isImage(const llvm::Value &val) {
225 return llvm::isImageReadOnly(val) || llvm::isImageWriteOnly(val) ||
226 llvm::isImageReadWrite(val);
227 }
228
isManaged(const llvm::Value & val)229 bool llvm::isManaged(const llvm::Value &val) {
230 if(const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
231 unsigned annot;
232 if(llvm::findOneNVVMAnnotation(gv,
233 llvm::PropertyAnnotationNames[llvm::PROPERTY_MANAGED],
234 annot)) {
235 assert((annot == 1) && "Unexpected annotation on a managed symbol");
236 return true;
237 }
238 }
239 return false;
240 }
241
getTextureName(const llvm::Value & val)242 std::string llvm::getTextureName(const llvm::Value &val) {
243 assert(val.hasName() && "Found texture variable with no name");
244 return val.getName();
245 }
246
getSurfaceName(const llvm::Value & val)247 std::string llvm::getSurfaceName(const llvm::Value &val) {
248 assert(val.hasName() && "Found surface variable with no name");
249 return val.getName();
250 }
251
getSamplerName(const llvm::Value & val)252 std::string llvm::getSamplerName(const llvm::Value &val) {
253 assert(val.hasName() && "Found sampler variable with no name");
254 return val.getName();
255 }
256
getMaxNTIDx(const Function & F,unsigned & x)257 bool llvm::getMaxNTIDx(const Function &F, unsigned &x) {
258 return (llvm::findOneNVVMAnnotation(
259 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_MAXNTID_X], x));
260 }
261
getMaxNTIDy(const Function & F,unsigned & y)262 bool llvm::getMaxNTIDy(const Function &F, unsigned &y) {
263 return (llvm::findOneNVVMAnnotation(
264 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_MAXNTID_Y], y));
265 }
266
getMaxNTIDz(const Function & F,unsigned & z)267 bool llvm::getMaxNTIDz(const Function &F, unsigned &z) {
268 return (llvm::findOneNVVMAnnotation(
269 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_MAXNTID_Z], z));
270 }
271
getReqNTIDx(const Function & F,unsigned & x)272 bool llvm::getReqNTIDx(const Function &F, unsigned &x) {
273 return (llvm::findOneNVVMAnnotation(
274 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_REQNTID_X], x));
275 }
276
getReqNTIDy(const Function & F,unsigned & y)277 bool llvm::getReqNTIDy(const Function &F, unsigned &y) {
278 return (llvm::findOneNVVMAnnotation(
279 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_REQNTID_Y], y));
280 }
281
getReqNTIDz(const Function & F,unsigned & z)282 bool llvm::getReqNTIDz(const Function &F, unsigned &z) {
283 return (llvm::findOneNVVMAnnotation(
284 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_REQNTID_Z], z));
285 }
286
getMinCTASm(const Function & F,unsigned & x)287 bool llvm::getMinCTASm(const Function &F, unsigned &x) {
288 return (llvm::findOneNVVMAnnotation(
289 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_MINNCTAPERSM], x));
290 }
291
isKernelFunction(const Function & F)292 bool llvm::isKernelFunction(const Function &F) {
293 unsigned x = 0;
294 bool retval = llvm::findOneNVVMAnnotation(
295 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_ISKERNEL_FUNCTION], x);
296 if (retval == false) {
297 // There is no NVVM metadata, check the calling convention
298 if (F.getCallingConv() == llvm::CallingConv::PTX_Kernel)
299 return true;
300 else
301 return false;
302 }
303 return (x == 1);
304 }
305
getAlign(const Function & F,unsigned index,unsigned & align)306 bool llvm::getAlign(const Function &F, unsigned index, unsigned &align) {
307 std::vector<unsigned> Vs;
308 bool retval = llvm::findAllNVVMAnnotation(
309 &F, llvm::PropertyAnnotationNames[llvm::PROPERTY_ALIGN], Vs);
310 if (retval == false)
311 return false;
312 for (int i = 0, e = Vs.size(); i < e; i++) {
313 unsigned v = Vs[i];
314 if ((v >> 16) == index) {
315 align = v & 0xFFFF;
316 return true;
317 }
318 }
319 return false;
320 }
321
getAlign(const CallInst & I,unsigned index,unsigned & align)322 bool llvm::getAlign(const CallInst &I, unsigned index, unsigned &align) {
323 if (MDNode *alignNode = I.getMetadata("callalign")) {
324 for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
325 if (const ConstantInt *CI =
326 mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {
327 unsigned v = CI->getZExtValue();
328 if ((v >> 16) == index) {
329 align = v & 0xFFFF;
330 return true;
331 }
332 if ((v >> 16) > index) {
333 return false;
334 }
335 }
336 }
337 }
338 return false;
339 }
340
isBarrierIntrinsic(Intrinsic::ID id)341 bool llvm::isBarrierIntrinsic(Intrinsic::ID id) {
342 if ((id == Intrinsic::nvvm_barrier0) ||
343 (id == Intrinsic::nvvm_barrier0_popc) ||
344 (id == Intrinsic::nvvm_barrier0_and) ||
345 (id == Intrinsic::nvvm_barrier0_or) ||
346 (id == Intrinsic::cuda_syncthreads))
347 return true;
348 return false;
349 }
350
351 // Interface for checking all memory space transfer related intrinsics
isMemorySpaceTransferIntrinsic(Intrinsic::ID id)352 bool llvm::isMemorySpaceTransferIntrinsic(Intrinsic::ID id) {
353 if (id == Intrinsic::nvvm_ptr_local_to_gen ||
354 id == Intrinsic::nvvm_ptr_shared_to_gen ||
355 id == Intrinsic::nvvm_ptr_global_to_gen ||
356 id == Intrinsic::nvvm_ptr_constant_to_gen ||
357 id == Intrinsic::nvvm_ptr_gen_to_global ||
358 id == Intrinsic::nvvm_ptr_gen_to_shared ||
359 id == Intrinsic::nvvm_ptr_gen_to_local ||
360 id == Intrinsic::nvvm_ptr_gen_to_constant ||
361 id == Intrinsic::nvvm_ptr_gen_to_param) {
362 return true;
363 }
364
365 return false;
366 }
367
368 // consider several special intrinsics in striping pointer casts, and
369 // provide an option to ignore GEP indicies for find out the base address only
370 // which could be used in simple alias disambigurate.
371 const Value *
skipPointerTransfer(const Value * V,bool ignore_GEP_indices)372 llvm::skipPointerTransfer(const Value *V, bool ignore_GEP_indices) {
373 V = V->stripPointerCasts();
374 while (true) {
375 if (const IntrinsicInst *IS = dyn_cast<IntrinsicInst>(V)) {
376 if (isMemorySpaceTransferIntrinsic(IS->getIntrinsicID())) {
377 V = IS->getArgOperand(0)->stripPointerCasts();
378 continue;
379 }
380 } else if (ignore_GEP_indices)
381 if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
382 V = GEP->getPointerOperand()->stripPointerCasts();
383 continue;
384 }
385 break;
386 }
387 return V;
388 }
389
390 // consider several special intrinsics in striping pointer casts, and
391 // - ignore GEP indicies for find out the base address only, and
392 // - tracking PHINode
393 // which could be used in simple alias disambigurate.
394 const Value *
skipPointerTransfer(const Value * V,std::set<const Value * > & processed)395 llvm::skipPointerTransfer(const Value *V, std::set<const Value *> &processed) {
396 if (processed.find(V) != processed.end())
397 return nullptr;
398 processed.insert(V);
399
400 const Value *V2 = V->stripPointerCasts();
401 if (V2 != V && processed.find(V2) != processed.end())
402 return nullptr;
403 processed.insert(V2);
404
405 V = V2;
406
407 while (true) {
408 if (const IntrinsicInst *IS = dyn_cast<IntrinsicInst>(V)) {
409 if (isMemorySpaceTransferIntrinsic(IS->getIntrinsicID())) {
410 V = IS->getArgOperand(0)->stripPointerCasts();
411 continue;
412 }
413 } else if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V)) {
414 V = GEP->getPointerOperand()->stripPointerCasts();
415 continue;
416 } else if (const PHINode *PN = dyn_cast<PHINode>(V)) {
417 if (V != V2 && processed.find(V) != processed.end())
418 return nullptr;
419 processed.insert(PN);
420 const Value *common = nullptr;
421 for (unsigned i = 0; i != PN->getNumIncomingValues(); ++i) {
422 const Value *pv = PN->getIncomingValue(i);
423 const Value *base = skipPointerTransfer(pv, processed);
424 if (base) {
425 if (!common)
426 common = base;
427 else if (common != base)
428 return PN;
429 }
430 }
431 if (!common)
432 return PN;
433 V = common;
434 }
435 break;
436 }
437 return V;
438 }
439
440 // The following are some useful utilities for debuggung
441
getParentBlock(Value * v)442 BasicBlock *llvm::getParentBlock(Value *v) {
443 if (BasicBlock *B = dyn_cast<BasicBlock>(v))
444 return B;
445
446 if (Instruction *I = dyn_cast<Instruction>(v))
447 return I->getParent();
448
449 return nullptr;
450 }
451
getParentFunction(Value * v)452 Function *llvm::getParentFunction(Value *v) {
453 if (Function *F = dyn_cast<Function>(v))
454 return F;
455
456 if (Instruction *I = dyn_cast<Instruction>(v))
457 return I->getParent()->getParent();
458
459 if (BasicBlock *B = dyn_cast<BasicBlock>(v))
460 return B->getParent();
461
462 return nullptr;
463 }
464
465 // Dump a block by name
dumpBlock(Value * v,char * blockName)466 void llvm::dumpBlock(Value *v, char *blockName) {
467 Function *F = getParentFunction(v);
468 if (!F)
469 return;
470
471 for (Function::iterator it = F->begin(), ie = F->end(); it != ie; ++it) {
472 BasicBlock *B = it;
473 if (strcmp(B->getName().data(), blockName) == 0) {
474 B->dump();
475 return;
476 }
477 }
478 }
479
480 // Find an instruction by name
getInst(Value * base,char * instName)481 Instruction *llvm::getInst(Value *base, char *instName) {
482 Function *F = getParentFunction(base);
483 if (!F)
484 return nullptr;
485
486 for (inst_iterator it = inst_begin(F), ie = inst_end(F); it != ie; ++it) {
487 Instruction *I = &*it;
488 if (strcmp(I->getName().data(), instName) == 0) {
489 return I;
490 }
491 }
492
493 return nullptr;
494 }
495
496 // Dump an instruction by nane
dumpInst(Value * base,char * instName)497 void llvm::dumpInst(Value *base, char *instName) {
498 Instruction *I = getInst(base, instName);
499 if (I)
500 I->dump();
501 }
502
503 // Dump an instruction and all dependent instructions
dumpInstRec(Value * v,std::set<Instruction * > * visited)504 void llvm::dumpInstRec(Value *v, std::set<Instruction *> *visited) {
505 if (Instruction *I = dyn_cast<Instruction>(v)) {
506
507 if (visited->find(I) != visited->end())
508 return;
509
510 visited->insert(I);
511
512 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
513 dumpInstRec(I->getOperand(i), visited);
514
515 I->dump();
516 }
517 }
518
519 // Dump an instruction and all dependent instructions
dumpInstRec(Value * v)520 void llvm::dumpInstRec(Value *v) {
521 std::set<Instruction *> visited;
522
523 //BasicBlock *B = getParentBlock(v);
524
525 dumpInstRec(v, &visited);
526 }
527
528 // Dump the parent for Instruction, block or function
dumpParent(Value * v)529 void llvm::dumpParent(Value *v) {
530 if (Instruction *I = dyn_cast<Instruction>(v)) {
531 I->getParent()->dump();
532 return;
533 }
534
535 if (BasicBlock *B = dyn_cast<BasicBlock>(v)) {
536 B->getParent()->dump();
537 return;
538 }
539
540 if (Function *F = dyn_cast<Function>(v)) {
541 F->getParent()->dump();
542 return;
543 }
544 }
545