1 //===-- InstrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements methods that make it really easy to deal with intrinsic
10 // functions.
11 //
12 // All intrinsic function calls are instances of the call instruction, so these
13 // are all subclasses of the CallInst class. Note that none of these classes
14 // has state or virtual methods, which is an important part of this gross/neat
15 // hack working.
16 //
17 // In some cases, arguments to intrinsics need to be generic and are defined as
18 // type pointer to empty struct { }*. To access the real item of interest the
19 // cast instruction needs to be stripped away.
20 //
21 //===----------------------------------------------------------------------===//
22
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/ADT/StringSwitch.h"
25 #include "llvm/IR/Constants.h"
26 #include "llvm/IR/DebugInfoMetadata.h"
27 #include "llvm/IR/GlobalVariable.h"
28 #include "llvm/IR/Metadata.h"
29 #include "llvm/IR/Module.h"
30 #include "llvm/IR/Operator.h"
31 #include "llvm/IR/PatternMatch.h"
32 #include "llvm/IR/Statepoint.h"
33
34 #include "llvm/Support/raw_ostream.h"
35 using namespace llvm;
36
37 //===----------------------------------------------------------------------===//
38 /// DbgVariableIntrinsic - This is the common base class for debug info
39 /// intrinsics for variables.
40 ///
41
42 iterator_range<DbgVariableIntrinsic::location_op_iterator>
location_ops() const43 DbgVariableIntrinsic::location_ops() const {
44 auto *MD = getRawLocation();
45 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
46
47 // If operand is ValueAsMetadata, return a range over just that operand.
48 if (auto *VAM = dyn_cast<ValueAsMetadata>(MD)) {
49 return {location_op_iterator(VAM), location_op_iterator(VAM + 1)};
50 }
51 // If operand is DIArgList, return a range over its args.
52 if (auto *AL = dyn_cast<DIArgList>(MD))
53 return {location_op_iterator(AL->args_begin()),
54 location_op_iterator(AL->args_end())};
55 // Operand must be an empty metadata tuple, so return empty iterator.
56 return {location_op_iterator(static_cast<ValueAsMetadata *>(nullptr)),
57 location_op_iterator(static_cast<ValueAsMetadata *>(nullptr))};
58 }
59
getVariableLocationOp(unsigned OpIdx) const60 Value *DbgVariableIntrinsic::getVariableLocationOp(unsigned OpIdx) const {
61 auto *MD = getRawLocation();
62 assert(MD && "First operand of DbgVariableIntrinsic should be non-null.");
63 if (auto *AL = dyn_cast<DIArgList>(MD))
64 return AL->getArgs()[OpIdx]->getValue();
65 if (isa<MDNode>(MD))
66 return nullptr;
67 assert(
68 isa<ValueAsMetadata>(MD) &&
69 "Attempted to get location operand from DbgVariableIntrinsic with none.");
70 auto *V = cast<ValueAsMetadata>(MD);
71 assert(OpIdx == 0 && "Operand Index must be 0 for a debug intrinsic with a "
72 "single location operand.");
73 return V->getValue();
74 }
75
getAsMetadata(Value * V)76 static ValueAsMetadata *getAsMetadata(Value *V) {
77 return isa<MetadataAsValue>(V) ? dyn_cast<ValueAsMetadata>(
78 cast<MetadataAsValue>(V)->getMetadata())
79 : ValueAsMetadata::get(V);
80 }
81
replaceVariableLocationOp(Value * OldValue,Value * NewValue)82 void DbgVariableIntrinsic::replaceVariableLocationOp(Value *OldValue,
83 Value *NewValue) {
84 assert(NewValue && "Values must be non-null");
85 auto Locations = location_ops();
86 auto OldIt = find(Locations, OldValue);
87 assert(OldIt != Locations.end() && "OldValue must be a current location");
88 if (!hasArgList()) {
89 Value *NewOperand = isa<MetadataAsValue>(NewValue)
90 ? NewValue
91 : MetadataAsValue::get(
92 getContext(), ValueAsMetadata::get(NewValue));
93 return setArgOperand(0, NewOperand);
94 }
95 SmallVector<ValueAsMetadata *, 4> MDs;
96 ValueAsMetadata *NewOperand = getAsMetadata(NewValue);
97 for (auto *VMD : Locations)
98 MDs.push_back(VMD == *OldIt ? NewOperand : getAsMetadata(VMD));
99 setArgOperand(
100 0, MetadataAsValue::get(getContext(), DIArgList::get(getContext(), MDs)));
101 }
replaceVariableLocationOp(unsigned OpIdx,Value * NewValue)102 void DbgVariableIntrinsic::replaceVariableLocationOp(unsigned OpIdx,
103 Value *NewValue) {
104 assert(OpIdx < getNumVariableLocationOps() && "Invalid Operand Index");
105 if (!hasArgList()) {
106 Value *NewOperand = isa<MetadataAsValue>(NewValue)
107 ? NewValue
108 : MetadataAsValue::get(
109 getContext(), ValueAsMetadata::get(NewValue));
110 return setArgOperand(0, NewOperand);
111 }
112 SmallVector<ValueAsMetadata *, 4> MDs;
113 ValueAsMetadata *NewOperand = getAsMetadata(NewValue);
114 for (unsigned Idx = 0; Idx < getNumVariableLocationOps(); ++Idx)
115 MDs.push_back(Idx == OpIdx ? NewOperand
116 : getAsMetadata(getVariableLocationOp(Idx)));
117 setArgOperand(
118 0, MetadataAsValue::get(getContext(), DIArgList::get(getContext(), MDs)));
119 }
120
addVariableLocationOps(ArrayRef<Value * > NewValues,DIExpression * NewExpr)121 void DbgVariableIntrinsic::addVariableLocationOps(ArrayRef<Value *> NewValues,
122 DIExpression *NewExpr) {
123 assert(NewExpr->hasAllLocationOps(getNumVariableLocationOps() +
124 NewValues.size()) &&
125 "NewExpr for debug variable intrinsic does not reference every "
126 "location operand.");
127 assert(!is_contained(NewValues, nullptr) && "New values must be non-null");
128 setArgOperand(2, MetadataAsValue::get(getContext(), NewExpr));
129 SmallVector<ValueAsMetadata *, 4> MDs;
130 for (auto *VMD : location_ops())
131 MDs.push_back(getAsMetadata(VMD));
132 for (auto *VMD : NewValues)
133 MDs.push_back(getAsMetadata(VMD));
134 setArgOperand(
135 0, MetadataAsValue::get(getContext(), DIArgList::get(getContext(), MDs)));
136 }
137
getFragmentSizeInBits() const138 Optional<uint64_t> DbgVariableIntrinsic::getFragmentSizeInBits() const {
139 if (auto Fragment = getExpression()->getFragmentInfo())
140 return Fragment->SizeInBits;
141 return getVariable()->getSizeInBits();
142 }
143
lookupLLVMIntrinsicByName(ArrayRef<const char * > NameTable,StringRef Name)144 int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
145 StringRef Name) {
146 assert(Name.startswith("llvm."));
147
148 // Do successive binary searches of the dotted name components. For
149 // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of
150 // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then
151 // "llvm.gc.experimental.statepoint", and then we will stop as the range is
152 // size 1. During the search, we can skip the prefix that we already know is
153 // identical. By using strncmp we consider names with differing suffixes to
154 // be part of the equal range.
155 size_t CmpEnd = 4; // Skip the "llvm" component.
156 const char *const *Low = NameTable.begin();
157 const char *const *High = NameTable.end();
158 const char *const *LastLow = Low;
159 while (CmpEnd < Name.size() && High - Low > 0) {
160 size_t CmpStart = CmpEnd;
161 CmpEnd = Name.find('.', CmpStart + 1);
162 CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
163 auto Cmp = [CmpStart, CmpEnd](const char *LHS, const char *RHS) {
164 return strncmp(LHS + CmpStart, RHS + CmpStart, CmpEnd - CmpStart) < 0;
165 };
166 LastLow = Low;
167 std::tie(Low, High) = std::equal_range(Low, High, Name.data(), Cmp);
168 }
169 if (High - Low > 0)
170 LastLow = Low;
171
172 if (LastLow == NameTable.end())
173 return -1;
174 StringRef NameFound = *LastLow;
175 if (Name == NameFound ||
176 (Name.startswith(NameFound) && Name[NameFound.size()] == '.'))
177 return LastLow - NameTable.begin();
178 return -1;
179 }
180
getStep() const181 Value *InstrProfIncrementInst::getStep() const {
182 if (InstrProfIncrementInstStep::classof(this)) {
183 return const_cast<Value *>(getArgOperand(4));
184 }
185 const Module *M = getModule();
186 LLVMContext &Context = M->getContext();
187 return ConstantInt::get(Type::getInt64Ty(Context), 1);
188 }
189
getRoundingMode() const190 Optional<RoundingMode> ConstrainedFPIntrinsic::getRoundingMode() const {
191 unsigned NumOperands = getNumArgOperands();
192 Metadata *MD = nullptr;
193 auto *MAV = dyn_cast<MetadataAsValue>(getArgOperand(NumOperands - 2));
194 if (MAV)
195 MD = MAV->getMetadata();
196 if (!MD || !isa<MDString>(MD))
197 return None;
198 return StrToRoundingMode(cast<MDString>(MD)->getString());
199 }
200
201 Optional<fp::ExceptionBehavior>
getExceptionBehavior() const202 ConstrainedFPIntrinsic::getExceptionBehavior() const {
203 unsigned NumOperands = getNumArgOperands();
204 Metadata *MD = nullptr;
205 auto *MAV = dyn_cast<MetadataAsValue>(getArgOperand(NumOperands - 1));
206 if (MAV)
207 MD = MAV->getMetadata();
208 if (!MD || !isa<MDString>(MD))
209 return None;
210 return StrToExceptionBehavior(cast<MDString>(MD)->getString());
211 }
212
isDefaultFPEnvironment() const213 bool ConstrainedFPIntrinsic::isDefaultFPEnvironment() const {
214 Optional<fp::ExceptionBehavior> Except = getExceptionBehavior();
215 if (Except) {
216 if (Except.getValue() != fp::ebIgnore)
217 return false;
218 }
219
220 Optional<RoundingMode> Rounding = getRoundingMode();
221 if (Rounding) {
222 if (Rounding.getValue() != RoundingMode::NearestTiesToEven)
223 return false;
224 }
225
226 return true;
227 }
228
getPredicate() const229 FCmpInst::Predicate ConstrainedFPCmpIntrinsic::getPredicate() const {
230 Metadata *MD = cast<MetadataAsValue>(getArgOperand(2))->getMetadata();
231 if (!MD || !isa<MDString>(MD))
232 return FCmpInst::BAD_FCMP_PREDICATE;
233 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
234 .Case("oeq", FCmpInst::FCMP_OEQ)
235 .Case("ogt", FCmpInst::FCMP_OGT)
236 .Case("oge", FCmpInst::FCMP_OGE)
237 .Case("olt", FCmpInst::FCMP_OLT)
238 .Case("ole", FCmpInst::FCMP_OLE)
239 .Case("one", FCmpInst::FCMP_ONE)
240 .Case("ord", FCmpInst::FCMP_ORD)
241 .Case("uno", FCmpInst::FCMP_UNO)
242 .Case("ueq", FCmpInst::FCMP_UEQ)
243 .Case("ugt", FCmpInst::FCMP_UGT)
244 .Case("uge", FCmpInst::FCMP_UGE)
245 .Case("ult", FCmpInst::FCMP_ULT)
246 .Case("ule", FCmpInst::FCMP_ULE)
247 .Case("une", FCmpInst::FCMP_UNE)
248 .Default(FCmpInst::BAD_FCMP_PREDICATE);
249 }
250
isUnaryOp() const251 bool ConstrainedFPIntrinsic::isUnaryOp() const {
252 switch (getIntrinsicID()) {
253 default:
254 return false;
255 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
256 case Intrinsic::INTRINSIC: \
257 return NARG == 1;
258 #include "llvm/IR/ConstrainedOps.def"
259 }
260 }
261
isTernaryOp() const262 bool ConstrainedFPIntrinsic::isTernaryOp() const {
263 switch (getIntrinsicID()) {
264 default:
265 return false;
266 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
267 case Intrinsic::INTRINSIC: \
268 return NARG == 3;
269 #include "llvm/IR/ConstrainedOps.def"
270 }
271 }
272
classof(const IntrinsicInst * I)273 bool ConstrainedFPIntrinsic::classof(const IntrinsicInst *I) {
274 switch (I->getIntrinsicID()) {
275 #define INSTRUCTION(NAME, NARGS, ROUND_MODE, INTRINSIC) \
276 case Intrinsic::INTRINSIC:
277 #include "llvm/IR/ConstrainedOps.def"
278 return true;
279 default:
280 return false;
281 }
282 }
283
getStaticVectorLength() const284 ElementCount VPIntrinsic::getStaticVectorLength() const {
285 auto GetVectorLengthOfType = [](const Type *T) -> ElementCount {
286 auto VT = cast<VectorType>(T);
287 auto ElemCount = VT->getElementCount();
288 return ElemCount;
289 };
290
291 auto VPMask = getMaskParam();
292 return GetVectorLengthOfType(VPMask->getType());
293 }
294
getMaskParam() const295 Value *VPIntrinsic::getMaskParam() const {
296 auto maskPos = GetMaskParamPos(getIntrinsicID());
297 if (maskPos)
298 return getArgOperand(maskPos.getValue());
299 return nullptr;
300 }
301
setMaskParam(Value * NewMask)302 void VPIntrinsic::setMaskParam(Value *NewMask) {
303 auto MaskPos = GetMaskParamPos(getIntrinsicID());
304 setArgOperand(*MaskPos, NewMask);
305 }
306
getVectorLengthParam() const307 Value *VPIntrinsic::getVectorLengthParam() const {
308 auto vlenPos = GetVectorLengthParamPos(getIntrinsicID());
309 if (vlenPos)
310 return getArgOperand(vlenPos.getValue());
311 return nullptr;
312 }
313
setVectorLengthParam(Value * NewEVL)314 void VPIntrinsic::setVectorLengthParam(Value *NewEVL) {
315 auto EVLPos = GetVectorLengthParamPos(getIntrinsicID());
316 setArgOperand(*EVLPos, NewEVL);
317 }
318
GetMaskParamPos(Intrinsic::ID IntrinsicID)319 Optional<int> VPIntrinsic::GetMaskParamPos(Intrinsic::ID IntrinsicID) {
320 switch (IntrinsicID) {
321 default:
322 return None;
323
324 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
325 case Intrinsic::VPID: \
326 return MASKPOS;
327 #include "llvm/IR/VPIntrinsics.def"
328 }
329 }
330
GetVectorLengthParamPos(Intrinsic::ID IntrinsicID)331 Optional<int> VPIntrinsic::GetVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
332 switch (IntrinsicID) {
333 default:
334 return None;
335
336 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
337 case Intrinsic::VPID: \
338 return VLENPOS;
339 #include "llvm/IR/VPIntrinsics.def"
340 }
341 }
342
IsVPIntrinsic(Intrinsic::ID ID)343 bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) {
344 switch (ID) {
345 default:
346 return false;
347
348 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS) \
349 case Intrinsic::VPID: \
350 break;
351 #include "llvm/IR/VPIntrinsics.def"
352 }
353 return true;
354 }
355
356 // Equivalent non-predicated opcode
GetFunctionalOpcodeForVP(Intrinsic::ID ID)357 Optional<unsigned> VPIntrinsic::GetFunctionalOpcodeForVP(Intrinsic::ID ID) {
358 Optional<unsigned> FunctionalOC;
359 switch (ID) {
360 default:
361 break;
362 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
363 #define HANDLE_VP_TO_OPC(OPC) FunctionalOC = Instruction::OPC;
364 #define END_REGISTER_VP_INTRINSIC(...) break;
365 #include "llvm/IR/VPIntrinsics.def"
366 }
367
368 return FunctionalOC;
369 }
370
GetForOpcode(unsigned IROPC)371 Intrinsic::ID VPIntrinsic::GetForOpcode(unsigned IROPC) {
372 switch (IROPC) {
373 default:
374 return Intrinsic::not_intrinsic;
375
376 #define HANDLE_VP_TO_OPC(OPC) case Instruction::OPC:
377 #define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
378 #include "llvm/IR/VPIntrinsics.def"
379 }
380 }
381
canIgnoreVectorLengthParam() const382 bool VPIntrinsic::canIgnoreVectorLengthParam() const {
383 using namespace PatternMatch;
384
385 ElementCount EC = getStaticVectorLength();
386
387 // No vlen param - no lanes masked-off by it.
388 auto *VLParam = getVectorLengthParam();
389 if (!VLParam)
390 return true;
391
392 // Note that the VP intrinsic causes undefined behavior if the Explicit Vector
393 // Length parameter is strictly greater-than the number of vector elements of
394 // the operation. This function returns true when this is detected statically
395 // in the IR.
396
397 // Check whether "W == vscale * EC.getKnownMinValue()"
398 if (EC.isScalable()) {
399 // Undig the DL
400 auto ParMod = this->getModule();
401 if (!ParMod)
402 return false;
403 const auto &DL = ParMod->getDataLayout();
404
405 // Compare vscale patterns
406 uint64_t VScaleFactor;
407 if (match(VLParam, m_c_Mul(m_ConstantInt(VScaleFactor), m_VScale(DL))))
408 return VScaleFactor >= EC.getKnownMinValue();
409 return (EC.getKnownMinValue() == 1) && match(VLParam, m_VScale(DL));
410 }
411
412 // standard SIMD operation
413 auto VLConst = dyn_cast<ConstantInt>(VLParam);
414 if (!VLConst)
415 return false;
416
417 uint64_t VLNum = VLConst->getZExtValue();
418 if (VLNum >= EC.getKnownMinValue())
419 return true;
420
421 return false;
422 }
423
getBinaryOp() const424 Instruction::BinaryOps BinaryOpIntrinsic::getBinaryOp() const {
425 switch (getIntrinsicID()) {
426 case Intrinsic::uadd_with_overflow:
427 case Intrinsic::sadd_with_overflow:
428 case Intrinsic::uadd_sat:
429 case Intrinsic::sadd_sat:
430 return Instruction::Add;
431 case Intrinsic::usub_with_overflow:
432 case Intrinsic::ssub_with_overflow:
433 case Intrinsic::usub_sat:
434 case Intrinsic::ssub_sat:
435 return Instruction::Sub;
436 case Intrinsic::umul_with_overflow:
437 case Intrinsic::smul_with_overflow:
438 return Instruction::Mul;
439 default:
440 llvm_unreachable("Invalid intrinsic");
441 }
442 }
443
isSigned() const444 bool BinaryOpIntrinsic::isSigned() const {
445 switch (getIntrinsicID()) {
446 case Intrinsic::sadd_with_overflow:
447 case Intrinsic::ssub_with_overflow:
448 case Intrinsic::smul_with_overflow:
449 case Intrinsic::sadd_sat:
450 case Intrinsic::ssub_sat:
451 return true;
452 default:
453 return false;
454 }
455 }
456
getNoWrapKind() const457 unsigned BinaryOpIntrinsic::getNoWrapKind() const {
458 if (isSigned())
459 return OverflowingBinaryOperator::NoSignedWrap;
460 else
461 return OverflowingBinaryOperator::NoUnsignedWrap;
462 }
463
getStatepoint() const464 const GCStatepointInst *GCProjectionInst::getStatepoint() const {
465 const Value *Token = getArgOperand(0);
466
467 // This takes care both of relocates for call statepoints and relocates
468 // on normal path of invoke statepoint.
469 if (!isa<LandingPadInst>(Token))
470 return cast<GCStatepointInst>(Token);
471
472 // This relocate is on exceptional path of an invoke statepoint
473 const BasicBlock *InvokeBB =
474 cast<Instruction>(Token)->getParent()->getUniquePredecessor();
475
476 assert(InvokeBB && "safepoints should have unique landingpads");
477 assert(InvokeBB->getTerminator() &&
478 "safepoint block should be well formed");
479
480 return cast<GCStatepointInst>(InvokeBB->getTerminator());
481 }
482
getBasePtr() const483 Value *GCRelocateInst::getBasePtr() const {
484 if (auto Opt = getStatepoint()->getOperandBundle(LLVMContext::OB_gc_live))
485 return *(Opt->Inputs.begin() + getBasePtrIndex());
486 return *(getStatepoint()->arg_begin() + getBasePtrIndex());
487 }
488
getDerivedPtr() const489 Value *GCRelocateInst::getDerivedPtr() const {
490 if (auto Opt = getStatepoint()->getOperandBundle(LLVMContext::OB_gc_live))
491 return *(Opt->Inputs.begin() + getDerivedPtrIndex());
492 return *(getStatepoint()->arg_begin() + getDerivedPtrIndex());
493 }
494