106f32e7eSjoerg //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
206f32e7eSjoerg //
306f32e7eSjoerg // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
406f32e7eSjoerg // See https://llvm.org/LICENSE.txt for license information.
506f32e7eSjoerg // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
606f32e7eSjoerg //
706f32e7eSjoerg //===----------------------------------------------------------------------===//
806f32e7eSjoerg //
906f32e7eSjoerg // This file implements the IRBuilder class, which is used as a convenient way
1006f32e7eSjoerg // to create LLVM instructions with a consistent and simplified interface.
1106f32e7eSjoerg //
1206f32e7eSjoerg //===----------------------------------------------------------------------===//
1306f32e7eSjoerg
1406f32e7eSjoerg #include "llvm/IR/IRBuilder.h"
1506f32e7eSjoerg #include "llvm/ADT/ArrayRef.h"
1606f32e7eSjoerg #include "llvm/ADT/None.h"
1706f32e7eSjoerg #include "llvm/IR/Constant.h"
1806f32e7eSjoerg #include "llvm/IR/Constants.h"
1906f32e7eSjoerg #include "llvm/IR/DerivedTypes.h"
2006f32e7eSjoerg #include "llvm/IR/Function.h"
2106f32e7eSjoerg #include "llvm/IR/GlobalValue.h"
2206f32e7eSjoerg #include "llvm/IR/GlobalVariable.h"
2306f32e7eSjoerg #include "llvm/IR/IntrinsicInst.h"
2406f32e7eSjoerg #include "llvm/IR/Intrinsics.h"
2506f32e7eSjoerg #include "llvm/IR/LLVMContext.h"
26*da58b97aSjoerg #include "llvm/IR/NoFolder.h"
2706f32e7eSjoerg #include "llvm/IR/Operator.h"
2806f32e7eSjoerg #include "llvm/IR/Statepoint.h"
2906f32e7eSjoerg #include "llvm/IR/Type.h"
3006f32e7eSjoerg #include "llvm/IR/Value.h"
3106f32e7eSjoerg #include "llvm/Support/Casting.h"
3206f32e7eSjoerg #include "llvm/Support/MathExtras.h"
3306f32e7eSjoerg #include <cassert>
3406f32e7eSjoerg #include <cstdint>
3506f32e7eSjoerg #include <vector>
3606f32e7eSjoerg
3706f32e7eSjoerg using namespace llvm;
3806f32e7eSjoerg
3906f32e7eSjoerg /// CreateGlobalString - Make a new global variable with an initializer that
4006f32e7eSjoerg /// has array of i8 type filled in with the nul terminated string value
4106f32e7eSjoerg /// specified. If Name is specified, it is the name of the global variable
4206f32e7eSjoerg /// created.
CreateGlobalString(StringRef Str,const Twine & Name,unsigned AddressSpace,Module * M)4306f32e7eSjoerg GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
4406f32e7eSjoerg const Twine &Name,
45*da58b97aSjoerg unsigned AddressSpace,
46*da58b97aSjoerg Module *M) {
4706f32e7eSjoerg Constant *StrConstant = ConstantDataArray::getString(Context, Str);
48*da58b97aSjoerg if (!M)
49*da58b97aSjoerg M = BB->getParent()->getParent();
50*da58b97aSjoerg auto *GV = new GlobalVariable(
51*da58b97aSjoerg *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
52*da58b97aSjoerg StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
5306f32e7eSjoerg GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
54*da58b97aSjoerg GV->setAlignment(Align(1));
5506f32e7eSjoerg return GV;
5606f32e7eSjoerg }
5706f32e7eSjoerg
getCurrentFunctionReturnType() const5806f32e7eSjoerg Type *IRBuilderBase::getCurrentFunctionReturnType() const {
5906f32e7eSjoerg assert(BB && BB->getParent() && "No current function!");
6006f32e7eSjoerg return BB->getParent()->getReturnType();
6106f32e7eSjoerg }
6206f32e7eSjoerg
getCastedInt8PtrValue(Value * Ptr)6306f32e7eSjoerg Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
6406f32e7eSjoerg auto *PT = cast<PointerType>(Ptr->getType());
6506f32e7eSjoerg if (PT->getElementType()->isIntegerTy(8))
6606f32e7eSjoerg return Ptr;
6706f32e7eSjoerg
6806f32e7eSjoerg // Otherwise, we need to insert a bitcast.
69*da58b97aSjoerg return CreateBitCast(Ptr, getInt8PtrTy(PT->getAddressSpace()));
7006f32e7eSjoerg }
7106f32e7eSjoerg
createCallHelper(Function * Callee,ArrayRef<Value * > Ops,IRBuilderBase * Builder,const Twine & Name="",Instruction * FMFSource=nullptr,ArrayRef<OperandBundleDef> OpBundles={})7206f32e7eSjoerg static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
7306f32e7eSjoerg IRBuilderBase *Builder,
7406f32e7eSjoerg const Twine &Name = "",
75*da58b97aSjoerg Instruction *FMFSource = nullptr,
76*da58b97aSjoerg ArrayRef<OperandBundleDef> OpBundles = {}) {
77*da58b97aSjoerg CallInst *CI = Builder->CreateCall(Callee, Ops, OpBundles, Name);
7806f32e7eSjoerg if (FMFSource)
7906f32e7eSjoerg CI->copyFastMathFlags(FMFSource);
8006f32e7eSjoerg return CI;
8106f32e7eSjoerg }
8206f32e7eSjoerg
CreateVScale(Constant * Scaling,const Twine & Name)83*da58b97aSjoerg Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
84*da58b97aSjoerg assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
85*da58b97aSjoerg if (cast<ConstantInt>(Scaling)->isZero())
86*da58b97aSjoerg return Scaling;
87*da58b97aSjoerg Module *M = GetInsertBlock()->getParent()->getParent();
88*da58b97aSjoerg Function *TheFn =
89*da58b97aSjoerg Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
90*da58b97aSjoerg CallInst *CI = createCallHelper(TheFn, {}, this, Name);
91*da58b97aSjoerg return cast<ConstantInt>(Scaling)->getSExtValue() == 1
92*da58b97aSjoerg ? CI
93*da58b97aSjoerg : CreateMul(CI, Scaling);
9406f32e7eSjoerg }
9506f32e7eSjoerg
CreateStepVector(Type * DstType,const Twine & Name)96*da58b97aSjoerg Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
97*da58b97aSjoerg if (isa<ScalableVectorType>(DstType))
98*da58b97aSjoerg return CreateIntrinsic(Intrinsic::experimental_stepvector, {DstType}, {},
99*da58b97aSjoerg nullptr, Name);
100*da58b97aSjoerg
101*da58b97aSjoerg Type *STy = DstType->getScalarType();
102*da58b97aSjoerg unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
103*da58b97aSjoerg
104*da58b97aSjoerg // Create a vector of consecutive numbers from zero to VF.
105*da58b97aSjoerg SmallVector<Constant *, 8> Indices;
106*da58b97aSjoerg for (unsigned i = 0; i < NumEls; ++i)
107*da58b97aSjoerg Indices.push_back(ConstantInt::get(STy, i));
108*da58b97aSjoerg
109*da58b97aSjoerg // Add the consecutive indices to the vector value.
110*da58b97aSjoerg return ConstantVector::get(Indices);
111*da58b97aSjoerg }
112*da58b97aSjoerg
CreateMemSet(Value * Ptr,Value * Val,Value * Size,MaybeAlign Align,bool isVolatile,MDNode * TBAATag,MDNode * ScopeTag,MDNode * NoAliasTag)113*da58b97aSjoerg CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
114*da58b97aSjoerg MaybeAlign Align, bool isVolatile,
115*da58b97aSjoerg MDNode *TBAATag, MDNode *ScopeTag,
11606f32e7eSjoerg MDNode *NoAliasTag) {
11706f32e7eSjoerg Ptr = getCastedInt8PtrValue(Ptr);
11806f32e7eSjoerg Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
11906f32e7eSjoerg Type *Tys[] = { Ptr->getType(), Size->getType() };
12006f32e7eSjoerg Module *M = BB->getParent()->getParent();
12106f32e7eSjoerg Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
12206f32e7eSjoerg
12306f32e7eSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
12406f32e7eSjoerg
125*da58b97aSjoerg if (Align)
126*da58b97aSjoerg cast<MemSetInst>(CI)->setDestAlignment(Align->value());
12706f32e7eSjoerg
12806f32e7eSjoerg // Set the TBAA info if present.
12906f32e7eSjoerg if (TBAATag)
13006f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
13106f32e7eSjoerg
13206f32e7eSjoerg if (ScopeTag)
13306f32e7eSjoerg CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
13406f32e7eSjoerg
13506f32e7eSjoerg if (NoAliasTag)
13606f32e7eSjoerg CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
13706f32e7eSjoerg
13806f32e7eSjoerg return CI;
13906f32e7eSjoerg }
14006f32e7eSjoerg
CreateElementUnorderedAtomicMemSet(Value * Ptr,Value * Val,Value * Size,Align Alignment,uint32_t ElementSize,MDNode * TBAATag,MDNode * ScopeTag,MDNode * NoAliasTag)14106f32e7eSjoerg CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
142*da58b97aSjoerg Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
14306f32e7eSjoerg MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
14406f32e7eSjoerg
14506f32e7eSjoerg Ptr = getCastedInt8PtrValue(Ptr);
14606f32e7eSjoerg Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
14706f32e7eSjoerg Type *Tys[] = {Ptr->getType(), Size->getType()};
14806f32e7eSjoerg Module *M = BB->getParent()->getParent();
14906f32e7eSjoerg Function *TheFn = Intrinsic::getDeclaration(
15006f32e7eSjoerg M, Intrinsic::memset_element_unordered_atomic, Tys);
15106f32e7eSjoerg
15206f32e7eSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
15306f32e7eSjoerg
154*da58b97aSjoerg cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
15506f32e7eSjoerg
15606f32e7eSjoerg // Set the TBAA info if present.
15706f32e7eSjoerg if (TBAATag)
15806f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
15906f32e7eSjoerg
16006f32e7eSjoerg if (ScopeTag)
16106f32e7eSjoerg CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
16206f32e7eSjoerg
16306f32e7eSjoerg if (NoAliasTag)
16406f32e7eSjoerg CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
16506f32e7eSjoerg
16606f32e7eSjoerg return CI;
16706f32e7eSjoerg }
16806f32e7eSjoerg
CreateMemTransferInst(Intrinsic::ID IntrID,Value * Dst,MaybeAlign DstAlign,Value * Src,MaybeAlign SrcAlign,Value * Size,bool isVolatile,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)169*da58b97aSjoerg CallInst *IRBuilderBase::CreateMemTransferInst(
170*da58b97aSjoerg Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
171*da58b97aSjoerg MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
17206f32e7eSjoerg MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
17306f32e7eSjoerg Dst = getCastedInt8PtrValue(Dst);
17406f32e7eSjoerg Src = getCastedInt8PtrValue(Src);
17506f32e7eSjoerg
17606f32e7eSjoerg Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
17706f32e7eSjoerg Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
17806f32e7eSjoerg Module *M = BB->getParent()->getParent();
179*da58b97aSjoerg Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
18006f32e7eSjoerg
18106f32e7eSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
18206f32e7eSjoerg
183*da58b97aSjoerg auto* MCI = cast<MemTransferInst>(CI);
184*da58b97aSjoerg if (DstAlign)
185*da58b97aSjoerg MCI->setDestAlignment(*DstAlign);
186*da58b97aSjoerg if (SrcAlign)
187*da58b97aSjoerg MCI->setSourceAlignment(*SrcAlign);
18806f32e7eSjoerg
18906f32e7eSjoerg // Set the TBAA info if present.
19006f32e7eSjoerg if (TBAATag)
19106f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
19206f32e7eSjoerg
19306f32e7eSjoerg // Set the TBAA Struct info if present.
19406f32e7eSjoerg if (TBAAStructTag)
19506f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
19606f32e7eSjoerg
19706f32e7eSjoerg if (ScopeTag)
19806f32e7eSjoerg CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
19906f32e7eSjoerg
20006f32e7eSjoerg if (NoAliasTag)
20106f32e7eSjoerg CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
20206f32e7eSjoerg
20306f32e7eSjoerg return CI;
20406f32e7eSjoerg }
20506f32e7eSjoerg
CreateMemCpyInline(Value * Dst,MaybeAlign DstAlign,Value * Src,MaybeAlign SrcAlign,Value * Size)206*da58b97aSjoerg CallInst *IRBuilderBase::CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign,
207*da58b97aSjoerg Value *Src, MaybeAlign SrcAlign,
208*da58b97aSjoerg Value *Size) {
209*da58b97aSjoerg Dst = getCastedInt8PtrValue(Dst);
210*da58b97aSjoerg Src = getCastedInt8PtrValue(Src);
211*da58b97aSjoerg Value *IsVolatile = getInt1(false);
212*da58b97aSjoerg
213*da58b97aSjoerg Value *Ops[] = {Dst, Src, Size, IsVolatile};
214*da58b97aSjoerg Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
215*da58b97aSjoerg Function *F = BB->getParent();
216*da58b97aSjoerg Module *M = F->getParent();
217*da58b97aSjoerg Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy_inline, Tys);
218*da58b97aSjoerg
219*da58b97aSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
220*da58b97aSjoerg
221*da58b97aSjoerg auto *MCI = cast<MemCpyInlineInst>(CI);
222*da58b97aSjoerg if (DstAlign)
223*da58b97aSjoerg MCI->setDestAlignment(*DstAlign);
224*da58b97aSjoerg if (SrcAlign)
225*da58b97aSjoerg MCI->setSourceAlignment(*SrcAlign);
226*da58b97aSjoerg
227*da58b97aSjoerg return CI;
228*da58b97aSjoerg }
229*da58b97aSjoerg
CreateElementUnorderedAtomicMemCpy(Value * Dst,Align DstAlign,Value * Src,Align SrcAlign,Value * Size,uint32_t ElementSize,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)23006f32e7eSjoerg CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
231*da58b97aSjoerg Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
23206f32e7eSjoerg uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
23306f32e7eSjoerg MDNode *ScopeTag, MDNode *NoAliasTag) {
23406f32e7eSjoerg assert(DstAlign >= ElementSize &&
23506f32e7eSjoerg "Pointer alignment must be at least element size");
23606f32e7eSjoerg assert(SrcAlign >= ElementSize &&
23706f32e7eSjoerg "Pointer alignment must be at least element size");
23806f32e7eSjoerg Dst = getCastedInt8PtrValue(Dst);
23906f32e7eSjoerg Src = getCastedInt8PtrValue(Src);
24006f32e7eSjoerg
24106f32e7eSjoerg Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
24206f32e7eSjoerg Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
24306f32e7eSjoerg Module *M = BB->getParent()->getParent();
24406f32e7eSjoerg Function *TheFn = Intrinsic::getDeclaration(
24506f32e7eSjoerg M, Intrinsic::memcpy_element_unordered_atomic, Tys);
24606f32e7eSjoerg
24706f32e7eSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
24806f32e7eSjoerg
24906f32e7eSjoerg // Set the alignment of the pointer args.
25006f32e7eSjoerg auto *AMCI = cast<AtomicMemCpyInst>(CI);
25106f32e7eSjoerg AMCI->setDestAlignment(DstAlign);
25206f32e7eSjoerg AMCI->setSourceAlignment(SrcAlign);
25306f32e7eSjoerg
25406f32e7eSjoerg // Set the TBAA info if present.
25506f32e7eSjoerg if (TBAATag)
25606f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
25706f32e7eSjoerg
25806f32e7eSjoerg // Set the TBAA Struct info if present.
25906f32e7eSjoerg if (TBAAStructTag)
26006f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
26106f32e7eSjoerg
26206f32e7eSjoerg if (ScopeTag)
26306f32e7eSjoerg CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
26406f32e7eSjoerg
26506f32e7eSjoerg if (NoAliasTag)
26606f32e7eSjoerg CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
26706f32e7eSjoerg
26806f32e7eSjoerg return CI;
26906f32e7eSjoerg }
27006f32e7eSjoerg
CreateMemMove(Value * Dst,MaybeAlign DstAlign,Value * Src,MaybeAlign SrcAlign,Value * Size,bool isVolatile,MDNode * TBAATag,MDNode * ScopeTag,MDNode * NoAliasTag)271*da58b97aSjoerg CallInst *IRBuilderBase::CreateMemMove(Value *Dst, MaybeAlign DstAlign,
272*da58b97aSjoerg Value *Src, MaybeAlign SrcAlign,
273*da58b97aSjoerg Value *Size, bool isVolatile,
274*da58b97aSjoerg MDNode *TBAATag, MDNode *ScopeTag,
27506f32e7eSjoerg MDNode *NoAliasTag) {
27606f32e7eSjoerg Dst = getCastedInt8PtrValue(Dst);
27706f32e7eSjoerg Src = getCastedInt8PtrValue(Src);
27806f32e7eSjoerg
27906f32e7eSjoerg Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
28006f32e7eSjoerg Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
28106f32e7eSjoerg Module *M = BB->getParent()->getParent();
28206f32e7eSjoerg Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
28306f32e7eSjoerg
28406f32e7eSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
28506f32e7eSjoerg
28606f32e7eSjoerg auto *MMI = cast<MemMoveInst>(CI);
287*da58b97aSjoerg if (DstAlign)
288*da58b97aSjoerg MMI->setDestAlignment(*DstAlign);
289*da58b97aSjoerg if (SrcAlign)
290*da58b97aSjoerg MMI->setSourceAlignment(*SrcAlign);
29106f32e7eSjoerg
29206f32e7eSjoerg // Set the TBAA info if present.
29306f32e7eSjoerg if (TBAATag)
29406f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
29506f32e7eSjoerg
29606f32e7eSjoerg if (ScopeTag)
29706f32e7eSjoerg CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
29806f32e7eSjoerg
29906f32e7eSjoerg if (NoAliasTag)
30006f32e7eSjoerg CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
30106f32e7eSjoerg
30206f32e7eSjoerg return CI;
30306f32e7eSjoerg }
30406f32e7eSjoerg
CreateElementUnorderedAtomicMemMove(Value * Dst,Align DstAlign,Value * Src,Align SrcAlign,Value * Size,uint32_t ElementSize,MDNode * TBAATag,MDNode * TBAAStructTag,MDNode * ScopeTag,MDNode * NoAliasTag)30506f32e7eSjoerg CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
306*da58b97aSjoerg Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
30706f32e7eSjoerg uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
30806f32e7eSjoerg MDNode *ScopeTag, MDNode *NoAliasTag) {
30906f32e7eSjoerg assert(DstAlign >= ElementSize &&
31006f32e7eSjoerg "Pointer alignment must be at least element size");
31106f32e7eSjoerg assert(SrcAlign >= ElementSize &&
31206f32e7eSjoerg "Pointer alignment must be at least element size");
31306f32e7eSjoerg Dst = getCastedInt8PtrValue(Dst);
31406f32e7eSjoerg Src = getCastedInt8PtrValue(Src);
31506f32e7eSjoerg
31606f32e7eSjoerg Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
31706f32e7eSjoerg Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
31806f32e7eSjoerg Module *M = BB->getParent()->getParent();
31906f32e7eSjoerg Function *TheFn = Intrinsic::getDeclaration(
32006f32e7eSjoerg M, Intrinsic::memmove_element_unordered_atomic, Tys);
32106f32e7eSjoerg
32206f32e7eSjoerg CallInst *CI = createCallHelper(TheFn, Ops, this);
32306f32e7eSjoerg
32406f32e7eSjoerg // Set the alignment of the pointer args.
325*da58b97aSjoerg CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
326*da58b97aSjoerg CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
32706f32e7eSjoerg
32806f32e7eSjoerg // Set the TBAA info if present.
32906f32e7eSjoerg if (TBAATag)
33006f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
33106f32e7eSjoerg
33206f32e7eSjoerg // Set the TBAA Struct info if present.
33306f32e7eSjoerg if (TBAAStructTag)
33406f32e7eSjoerg CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
33506f32e7eSjoerg
33606f32e7eSjoerg if (ScopeTag)
33706f32e7eSjoerg CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
33806f32e7eSjoerg
33906f32e7eSjoerg if (NoAliasTag)
34006f32e7eSjoerg CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
34106f32e7eSjoerg
34206f32e7eSjoerg return CI;
34306f32e7eSjoerg }
34406f32e7eSjoerg
getReductionIntrinsic(IRBuilderBase * Builder,Intrinsic::ID ID,Value * Src)34506f32e7eSjoerg static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
34606f32e7eSjoerg Value *Src) {
34706f32e7eSjoerg Module *M = Builder->GetInsertBlock()->getParent()->getParent();
34806f32e7eSjoerg Value *Ops[] = {Src};
34906f32e7eSjoerg Type *Tys[] = { Src->getType() };
35006f32e7eSjoerg auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
35106f32e7eSjoerg return createCallHelper(Decl, Ops, Builder);
35206f32e7eSjoerg }
35306f32e7eSjoerg
CreateFAddReduce(Value * Acc,Value * Src)35406f32e7eSjoerg CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
35506f32e7eSjoerg Module *M = GetInsertBlock()->getParent()->getParent();
35606f32e7eSjoerg Value *Ops[] = {Acc, Src};
357*da58b97aSjoerg auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
358*da58b97aSjoerg {Src->getType()});
35906f32e7eSjoerg return createCallHelper(Decl, Ops, this);
36006f32e7eSjoerg }
36106f32e7eSjoerg
CreateFMulReduce(Value * Acc,Value * Src)36206f32e7eSjoerg CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
36306f32e7eSjoerg Module *M = GetInsertBlock()->getParent()->getParent();
36406f32e7eSjoerg Value *Ops[] = {Acc, Src};
365*da58b97aSjoerg auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
366*da58b97aSjoerg {Src->getType()});
36706f32e7eSjoerg return createCallHelper(Decl, Ops, this);
36806f32e7eSjoerg }
36906f32e7eSjoerg
CreateAddReduce(Value * Src)37006f32e7eSjoerg CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
371*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_add, Src);
37206f32e7eSjoerg }
37306f32e7eSjoerg
CreateMulReduce(Value * Src)37406f32e7eSjoerg CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
375*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_mul, Src);
37606f32e7eSjoerg }
37706f32e7eSjoerg
CreateAndReduce(Value * Src)37806f32e7eSjoerg CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
379*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_and, Src);
38006f32e7eSjoerg }
38106f32e7eSjoerg
CreateOrReduce(Value * Src)38206f32e7eSjoerg CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
383*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_or, Src);
38406f32e7eSjoerg }
38506f32e7eSjoerg
CreateXorReduce(Value * Src)38606f32e7eSjoerg CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
387*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_xor, Src);
38806f32e7eSjoerg }
38906f32e7eSjoerg
CreateIntMaxReduce(Value * Src,bool IsSigned)39006f32e7eSjoerg CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
391*da58b97aSjoerg auto ID =
392*da58b97aSjoerg IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
39306f32e7eSjoerg return getReductionIntrinsic(this, ID, Src);
39406f32e7eSjoerg }
39506f32e7eSjoerg
CreateIntMinReduce(Value * Src,bool IsSigned)39606f32e7eSjoerg CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
397*da58b97aSjoerg auto ID =
398*da58b97aSjoerg IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
39906f32e7eSjoerg return getReductionIntrinsic(this, ID, Src);
40006f32e7eSjoerg }
40106f32e7eSjoerg
CreateFPMaxReduce(Value * Src)402*da58b97aSjoerg CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
403*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmax, Src);
40406f32e7eSjoerg }
40506f32e7eSjoerg
CreateFPMinReduce(Value * Src)406*da58b97aSjoerg CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
407*da58b97aSjoerg return getReductionIntrinsic(this, Intrinsic::vector_reduce_fmin, Src);
40806f32e7eSjoerg }
40906f32e7eSjoerg
CreateLifetimeStart(Value * Ptr,ConstantInt * Size)41006f32e7eSjoerg CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
41106f32e7eSjoerg assert(isa<PointerType>(Ptr->getType()) &&
41206f32e7eSjoerg "lifetime.start only applies to pointers.");
41306f32e7eSjoerg Ptr = getCastedInt8PtrValue(Ptr);
41406f32e7eSjoerg if (!Size)
41506f32e7eSjoerg Size = getInt64(-1);
41606f32e7eSjoerg else
41706f32e7eSjoerg assert(Size->getType() == getInt64Ty() &&
41806f32e7eSjoerg "lifetime.start requires the size to be an i64");
41906f32e7eSjoerg Value *Ops[] = { Size, Ptr };
42006f32e7eSjoerg Module *M = BB->getParent()->getParent();
42106f32e7eSjoerg Function *TheFn =
42206f32e7eSjoerg Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
42306f32e7eSjoerg return createCallHelper(TheFn, Ops, this);
42406f32e7eSjoerg }
42506f32e7eSjoerg
CreateLifetimeEnd(Value * Ptr,ConstantInt * Size)42606f32e7eSjoerg CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
42706f32e7eSjoerg assert(isa<PointerType>(Ptr->getType()) &&
42806f32e7eSjoerg "lifetime.end only applies to pointers.");
42906f32e7eSjoerg Ptr = getCastedInt8PtrValue(Ptr);
43006f32e7eSjoerg if (!Size)
43106f32e7eSjoerg Size = getInt64(-1);
43206f32e7eSjoerg else
43306f32e7eSjoerg assert(Size->getType() == getInt64Ty() &&
43406f32e7eSjoerg "lifetime.end requires the size to be an i64");
43506f32e7eSjoerg Value *Ops[] = { Size, Ptr };
43606f32e7eSjoerg Module *M = BB->getParent()->getParent();
43706f32e7eSjoerg Function *TheFn =
43806f32e7eSjoerg Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
43906f32e7eSjoerg return createCallHelper(TheFn, Ops, this);
44006f32e7eSjoerg }
44106f32e7eSjoerg
CreateInvariantStart(Value * Ptr,ConstantInt * Size)44206f32e7eSjoerg CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
44306f32e7eSjoerg
44406f32e7eSjoerg assert(isa<PointerType>(Ptr->getType()) &&
44506f32e7eSjoerg "invariant.start only applies to pointers.");
44606f32e7eSjoerg Ptr = getCastedInt8PtrValue(Ptr);
44706f32e7eSjoerg if (!Size)
44806f32e7eSjoerg Size = getInt64(-1);
44906f32e7eSjoerg else
45006f32e7eSjoerg assert(Size->getType() == getInt64Ty() &&
45106f32e7eSjoerg "invariant.start requires the size to be an i64");
45206f32e7eSjoerg
45306f32e7eSjoerg Value *Ops[] = {Size, Ptr};
45406f32e7eSjoerg // Fill in the single overloaded type: memory object type.
45506f32e7eSjoerg Type *ObjectPtr[1] = {Ptr->getType()};
45606f32e7eSjoerg Module *M = BB->getParent()->getParent();
45706f32e7eSjoerg Function *TheFn =
45806f32e7eSjoerg Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
45906f32e7eSjoerg return createCallHelper(TheFn, Ops, this);
46006f32e7eSjoerg }
46106f32e7eSjoerg
462*da58b97aSjoerg CallInst *
CreateAssumption(Value * Cond,ArrayRef<OperandBundleDef> OpBundles)463*da58b97aSjoerg IRBuilderBase::CreateAssumption(Value *Cond,
464*da58b97aSjoerg ArrayRef<OperandBundleDef> OpBundles) {
46506f32e7eSjoerg assert(Cond->getType() == getInt1Ty() &&
46606f32e7eSjoerg "an assumption condition must be of type i1");
46706f32e7eSjoerg
46806f32e7eSjoerg Value *Ops[] = { Cond };
46906f32e7eSjoerg Module *M = BB->getParent()->getParent();
47006f32e7eSjoerg Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
471*da58b97aSjoerg return createCallHelper(FnAssume, Ops, this, "", nullptr, OpBundles);
472*da58b97aSjoerg }
473*da58b97aSjoerg
CreateNoAliasScopeDeclaration(Value * Scope)474*da58b97aSjoerg Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
475*da58b97aSjoerg Module *M = BB->getModule();
476*da58b97aSjoerg auto *FnIntrinsic = Intrinsic::getDeclaration(
477*da58b97aSjoerg M, Intrinsic::experimental_noalias_scope_decl, {});
478*da58b97aSjoerg return createCallHelper(FnIntrinsic, {Scope}, this);
47906f32e7eSjoerg }
48006f32e7eSjoerg
48106f32e7eSjoerg /// Create a call to a Masked Load intrinsic.
48206f32e7eSjoerg /// \p Ptr - base pointer for the load
483*da58b97aSjoerg /// \p Alignment - alignment of the source location
48406f32e7eSjoerg /// \p Mask - vector of booleans which indicates what vector lanes should
48506f32e7eSjoerg /// be accessed in memory
48606f32e7eSjoerg /// \p PassThru - pass-through value that is used to fill the masked-off lanes
48706f32e7eSjoerg /// of the result
48806f32e7eSjoerg /// \p Name - name of the result variable
CreateMaskedLoad(Value * Ptr,Align Alignment,Value * Mask,Value * PassThru,const Twine & Name)489*da58b97aSjoerg CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, Align Alignment,
49006f32e7eSjoerg Value *Mask, Value *PassThru,
49106f32e7eSjoerg const Twine &Name) {
49206f32e7eSjoerg auto *PtrTy = cast<PointerType>(Ptr->getType());
49306f32e7eSjoerg Type *DataTy = PtrTy->getElementType();
49406f32e7eSjoerg assert(DataTy->isVectorTy() && "Ptr should point to a vector");
49506f32e7eSjoerg assert(Mask && "Mask should not be all-ones (null)");
49606f32e7eSjoerg if (!PassThru)
49706f32e7eSjoerg PassThru = UndefValue::get(DataTy);
49806f32e7eSjoerg Type *OverloadedTypes[] = { DataTy, PtrTy };
499*da58b97aSjoerg Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
50006f32e7eSjoerg return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
50106f32e7eSjoerg OverloadedTypes, Name);
50206f32e7eSjoerg }
50306f32e7eSjoerg
50406f32e7eSjoerg /// Create a call to a Masked Store intrinsic.
50506f32e7eSjoerg /// \p Val - data to be stored,
50606f32e7eSjoerg /// \p Ptr - base pointer for the store
507*da58b97aSjoerg /// \p Alignment - alignment of the destination location
50806f32e7eSjoerg /// \p Mask - vector of booleans which indicates what vector lanes should
50906f32e7eSjoerg /// be accessed in memory
CreateMaskedStore(Value * Val,Value * Ptr,Align Alignment,Value * Mask)51006f32e7eSjoerg CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
511*da58b97aSjoerg Align Alignment, Value *Mask) {
51206f32e7eSjoerg auto *PtrTy = cast<PointerType>(Ptr->getType());
51306f32e7eSjoerg Type *DataTy = PtrTy->getElementType();
51406f32e7eSjoerg assert(DataTy->isVectorTy() && "Ptr should point to a vector");
51506f32e7eSjoerg assert(Mask && "Mask should not be all-ones (null)");
51606f32e7eSjoerg Type *OverloadedTypes[] = { DataTy, PtrTy };
517*da58b97aSjoerg Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
51806f32e7eSjoerg return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
51906f32e7eSjoerg }
52006f32e7eSjoerg
52106f32e7eSjoerg /// Create a call to a Masked intrinsic, with given intrinsic Id,
52206f32e7eSjoerg /// an array of operands - Ops, and an array of overloaded types -
52306f32e7eSjoerg /// OverloadedTypes.
CreateMaskedIntrinsic(Intrinsic::ID Id,ArrayRef<Value * > Ops,ArrayRef<Type * > OverloadedTypes,const Twine & Name)52406f32e7eSjoerg CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
52506f32e7eSjoerg ArrayRef<Value *> Ops,
52606f32e7eSjoerg ArrayRef<Type *> OverloadedTypes,
52706f32e7eSjoerg const Twine &Name) {
52806f32e7eSjoerg Module *M = BB->getParent()->getParent();
52906f32e7eSjoerg Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
53006f32e7eSjoerg return createCallHelper(TheFn, Ops, this, Name);
53106f32e7eSjoerg }
53206f32e7eSjoerg
53306f32e7eSjoerg /// Create a call to a Masked Gather intrinsic.
53406f32e7eSjoerg /// \p Ptrs - vector of pointers for loading
53506f32e7eSjoerg /// \p Align - alignment for one element
53606f32e7eSjoerg /// \p Mask - vector of booleans which indicates what vector lanes should
53706f32e7eSjoerg /// be accessed in memory
53806f32e7eSjoerg /// \p PassThru - pass-through value that is used to fill the masked-off lanes
53906f32e7eSjoerg /// of the result
54006f32e7eSjoerg /// \p Name - name of the result variable
CreateMaskedGather(Value * Ptrs,Align Alignment,Value * Mask,Value * PassThru,const Twine & Name)541*da58b97aSjoerg CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, Align Alignment,
54206f32e7eSjoerg Value *Mask, Value *PassThru,
54306f32e7eSjoerg const Twine &Name) {
544*da58b97aSjoerg auto *PtrsTy = cast<VectorType>(Ptrs->getType());
545*da58b97aSjoerg auto *PtrTy = cast<PointerType>(PtrsTy->getElementType());
546*da58b97aSjoerg ElementCount NumElts = PtrsTy->getElementCount();
547*da58b97aSjoerg auto *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
54806f32e7eSjoerg
54906f32e7eSjoerg if (!Mask)
550*da58b97aSjoerg Mask = Constant::getAllOnesValue(
551*da58b97aSjoerg VectorType::get(Type::getInt1Ty(Context), NumElts));
55206f32e7eSjoerg
55306f32e7eSjoerg if (!PassThru)
55406f32e7eSjoerg PassThru = UndefValue::get(DataTy);
55506f32e7eSjoerg
55606f32e7eSjoerg Type *OverloadedTypes[] = {DataTy, PtrsTy};
557*da58b97aSjoerg Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
55806f32e7eSjoerg
55906f32e7eSjoerg // We specify only one type when we create this intrinsic. Types of other
56006f32e7eSjoerg // arguments are derived from this type.
56106f32e7eSjoerg return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
56206f32e7eSjoerg Name);
56306f32e7eSjoerg }
56406f32e7eSjoerg
56506f32e7eSjoerg /// Create a call to a Masked Scatter intrinsic.
56606f32e7eSjoerg /// \p Data - data to be stored,
56706f32e7eSjoerg /// \p Ptrs - the vector of pointers, where the \p Data elements should be
56806f32e7eSjoerg /// stored
56906f32e7eSjoerg /// \p Align - alignment for one element
57006f32e7eSjoerg /// \p Mask - vector of booleans which indicates what vector lanes should
57106f32e7eSjoerg /// be accessed in memory
CreateMaskedScatter(Value * Data,Value * Ptrs,Align Alignment,Value * Mask)57206f32e7eSjoerg CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
573*da58b97aSjoerg Align Alignment, Value *Mask) {
574*da58b97aSjoerg auto *PtrsTy = cast<VectorType>(Ptrs->getType());
575*da58b97aSjoerg auto *DataTy = cast<VectorType>(Data->getType());
576*da58b97aSjoerg ElementCount NumElts = PtrsTy->getElementCount();
57706f32e7eSjoerg
57806f32e7eSjoerg #ifndef NDEBUG
57906f32e7eSjoerg auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
580*da58b97aSjoerg assert(NumElts == DataTy->getElementCount() &&
58106f32e7eSjoerg PtrTy->getElementType() == DataTy->getElementType() &&
58206f32e7eSjoerg "Incompatible pointer and data types");
58306f32e7eSjoerg #endif
58406f32e7eSjoerg
58506f32e7eSjoerg if (!Mask)
586*da58b97aSjoerg Mask = Constant::getAllOnesValue(
587*da58b97aSjoerg VectorType::get(Type::getInt1Ty(Context), NumElts));
58806f32e7eSjoerg
58906f32e7eSjoerg Type *OverloadedTypes[] = {DataTy, PtrsTy};
590*da58b97aSjoerg Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
59106f32e7eSjoerg
59206f32e7eSjoerg // We specify only one type when we create this intrinsic. Types of other
59306f32e7eSjoerg // arguments are derived from this type.
59406f32e7eSjoerg return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
59506f32e7eSjoerg }
59606f32e7eSjoerg
597*da58b97aSjoerg template <typename T0>
59806f32e7eSjoerg static std::vector<Value *>
getStatepointArgs(IRBuilderBase & B,uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,uint32_t Flags,ArrayRef<T0> CallArgs)59906f32e7eSjoerg getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
600*da58b97aSjoerg Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
60106f32e7eSjoerg std::vector<Value *> Args;
60206f32e7eSjoerg Args.push_back(B.getInt64(ID));
60306f32e7eSjoerg Args.push_back(B.getInt32(NumPatchBytes));
60406f32e7eSjoerg Args.push_back(ActualCallee);
60506f32e7eSjoerg Args.push_back(B.getInt32(CallArgs.size()));
60606f32e7eSjoerg Args.push_back(B.getInt32(Flags));
607*da58b97aSjoerg llvm::append_range(Args, CallArgs);
608*da58b97aSjoerg // GC Transition and Deopt args are now always handled via operand bundle.
609*da58b97aSjoerg // They will be removed from the signature of gc.statepoint shortly.
610*da58b97aSjoerg Args.push_back(B.getInt32(0));
611*da58b97aSjoerg Args.push_back(B.getInt32(0));
612*da58b97aSjoerg // GC args are now encoded in the gc-live operand bundle
61306f32e7eSjoerg return Args;
61406f32e7eSjoerg }
61506f32e7eSjoerg
616*da58b97aSjoerg template<typename T1, typename T2, typename T3>
617*da58b97aSjoerg static std::vector<OperandBundleDef>
getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,Optional<ArrayRef<T2>> DeoptArgs,ArrayRef<T3> GCArgs)618*da58b97aSjoerg getStatepointBundles(Optional<ArrayRef<T1>> TransitionArgs,
619*da58b97aSjoerg Optional<ArrayRef<T2>> DeoptArgs,
620*da58b97aSjoerg ArrayRef<T3> GCArgs) {
621*da58b97aSjoerg std::vector<OperandBundleDef> Rval;
622*da58b97aSjoerg if (DeoptArgs) {
623*da58b97aSjoerg SmallVector<Value*, 16> DeoptValues;
624*da58b97aSjoerg llvm::append_range(DeoptValues, *DeoptArgs);
625*da58b97aSjoerg Rval.emplace_back("deopt", DeoptValues);
626*da58b97aSjoerg }
627*da58b97aSjoerg if (TransitionArgs) {
628*da58b97aSjoerg SmallVector<Value*, 16> TransitionValues;
629*da58b97aSjoerg llvm::append_range(TransitionValues, *TransitionArgs);
630*da58b97aSjoerg Rval.emplace_back("gc-transition", TransitionValues);
631*da58b97aSjoerg }
632*da58b97aSjoerg if (GCArgs.size()) {
633*da58b97aSjoerg SmallVector<Value*, 16> LiveValues;
634*da58b97aSjoerg llvm::append_range(LiveValues, GCArgs);
635*da58b97aSjoerg Rval.emplace_back("gc-live", LiveValues);
636*da58b97aSjoerg }
637*da58b97aSjoerg return Rval;
638*da58b97aSjoerg }
639*da58b97aSjoerg
64006f32e7eSjoerg template <typename T0, typename T1, typename T2, typename T3>
CreateGCStatepointCallCommon(IRBuilderBase * Builder,uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,uint32_t Flags,ArrayRef<T0> CallArgs,Optional<ArrayRef<T1>> TransitionArgs,Optional<ArrayRef<T2>> DeoptArgs,ArrayRef<T3> GCArgs,const Twine & Name)64106f32e7eSjoerg static CallInst *CreateGCStatepointCallCommon(
64206f32e7eSjoerg IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
64306f32e7eSjoerg Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
644*da58b97aSjoerg Optional<ArrayRef<T1>> TransitionArgs,
645*da58b97aSjoerg Optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
64606f32e7eSjoerg const Twine &Name) {
64706f32e7eSjoerg // Extract out the type of the callee.
64806f32e7eSjoerg auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
64906f32e7eSjoerg assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
65006f32e7eSjoerg "actual callee must be a callable value");
65106f32e7eSjoerg
65206f32e7eSjoerg Module *M = Builder->GetInsertBlock()->getParent()->getParent();
65306f32e7eSjoerg // Fill in the one generic type'd argument (the function is also vararg)
65406f32e7eSjoerg Type *ArgTypes[] = { FuncPtrType };
65506f32e7eSjoerg Function *FnStatepoint =
65606f32e7eSjoerg Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
65706f32e7eSjoerg ArgTypes);
65806f32e7eSjoerg
65906f32e7eSjoerg std::vector<Value *> Args =
66006f32e7eSjoerg getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
661*da58b97aSjoerg CallArgs);
662*da58b97aSjoerg
663*da58b97aSjoerg return Builder->CreateCall(FnStatepoint, Args,
664*da58b97aSjoerg getStatepointBundles(TransitionArgs, DeoptArgs,
665*da58b97aSjoerg GCArgs),
666*da58b97aSjoerg Name);
66706f32e7eSjoerg }
66806f32e7eSjoerg
CreateGCStatepointCall(uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,ArrayRef<Value * > CallArgs,Optional<ArrayRef<Value * >> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)66906f32e7eSjoerg CallInst *IRBuilderBase::CreateGCStatepointCall(
67006f32e7eSjoerg uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
671*da58b97aSjoerg ArrayRef<Value *> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
67206f32e7eSjoerg ArrayRef<Value *> GCArgs, const Twine &Name) {
67306f32e7eSjoerg return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
67406f32e7eSjoerg this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
67506f32e7eSjoerg CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
67606f32e7eSjoerg }
67706f32e7eSjoerg
CreateGCStatepointCall(uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,uint32_t Flags,ArrayRef<Value * > CallArgs,Optional<ArrayRef<Use>> TransitionArgs,Optional<ArrayRef<Use>> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)67806f32e7eSjoerg CallInst *IRBuilderBase::CreateGCStatepointCall(
67906f32e7eSjoerg uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
680*da58b97aSjoerg ArrayRef<Value *> CallArgs, Optional<ArrayRef<Use>> TransitionArgs,
681*da58b97aSjoerg Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
682*da58b97aSjoerg const Twine &Name) {
683*da58b97aSjoerg return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
68406f32e7eSjoerg this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
68506f32e7eSjoerg DeoptArgs, GCArgs, Name);
68606f32e7eSjoerg }
68706f32e7eSjoerg
CreateGCStatepointCall(uint64_t ID,uint32_t NumPatchBytes,Value * ActualCallee,ArrayRef<Use> CallArgs,Optional<ArrayRef<Value * >> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)68806f32e7eSjoerg CallInst *IRBuilderBase::CreateGCStatepointCall(
68906f32e7eSjoerg uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
690*da58b97aSjoerg ArrayRef<Use> CallArgs, Optional<ArrayRef<Value *>> DeoptArgs,
69106f32e7eSjoerg ArrayRef<Value *> GCArgs, const Twine &Name) {
69206f32e7eSjoerg return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
69306f32e7eSjoerg this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
69406f32e7eSjoerg CallArgs, None, DeoptArgs, GCArgs, Name);
69506f32e7eSjoerg }
69606f32e7eSjoerg
69706f32e7eSjoerg template <typename T0, typename T1, typename T2, typename T3>
CreateGCStatepointInvokeCommon(IRBuilderBase * Builder,uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,uint32_t Flags,ArrayRef<T0> InvokeArgs,Optional<ArrayRef<T1>> TransitionArgs,Optional<ArrayRef<T2>> DeoptArgs,ArrayRef<T3> GCArgs,const Twine & Name)69806f32e7eSjoerg static InvokeInst *CreateGCStatepointInvokeCommon(
69906f32e7eSjoerg IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
70006f32e7eSjoerg Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
701*da58b97aSjoerg uint32_t Flags, ArrayRef<T0> InvokeArgs,
702*da58b97aSjoerg Optional<ArrayRef<T1>> TransitionArgs, Optional<ArrayRef<T2>> DeoptArgs,
703*da58b97aSjoerg ArrayRef<T3> GCArgs, const Twine &Name) {
70406f32e7eSjoerg // Extract out the type of the callee.
70506f32e7eSjoerg auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
70606f32e7eSjoerg assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
70706f32e7eSjoerg "actual callee must be a callable value");
70806f32e7eSjoerg
70906f32e7eSjoerg Module *M = Builder->GetInsertBlock()->getParent()->getParent();
71006f32e7eSjoerg // Fill in the one generic type'd argument (the function is also vararg)
71106f32e7eSjoerg Function *FnStatepoint = Intrinsic::getDeclaration(
71206f32e7eSjoerg M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
71306f32e7eSjoerg
71406f32e7eSjoerg std::vector<Value *> Args =
71506f32e7eSjoerg getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
716*da58b97aSjoerg InvokeArgs);
717*da58b97aSjoerg
718*da58b97aSjoerg return Builder->CreateInvoke(FnStatepoint, NormalDest, UnwindDest, Args,
719*da58b97aSjoerg getStatepointBundles(TransitionArgs, DeoptArgs,
720*da58b97aSjoerg GCArgs),
72106f32e7eSjoerg Name);
72206f32e7eSjoerg }
72306f32e7eSjoerg
CreateGCStatepointInvoke(uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,ArrayRef<Value * > InvokeArgs,Optional<ArrayRef<Value * >> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)72406f32e7eSjoerg InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
72506f32e7eSjoerg uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
72606f32e7eSjoerg BasicBlock *NormalDest, BasicBlock *UnwindDest,
727*da58b97aSjoerg ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Value *>> DeoptArgs,
72806f32e7eSjoerg ArrayRef<Value *> GCArgs, const Twine &Name) {
72906f32e7eSjoerg return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
73006f32e7eSjoerg this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
73106f32e7eSjoerg uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
73206f32e7eSjoerg DeoptArgs, GCArgs, Name);
73306f32e7eSjoerg }
73406f32e7eSjoerg
CreateGCStatepointInvoke(uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,uint32_t Flags,ArrayRef<Value * > InvokeArgs,Optional<ArrayRef<Use>> TransitionArgs,Optional<ArrayRef<Use>> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)73506f32e7eSjoerg InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
73606f32e7eSjoerg uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
73706f32e7eSjoerg BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
738*da58b97aSjoerg ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
739*da58b97aSjoerg Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
740*da58b97aSjoerg return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
74106f32e7eSjoerg this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
74206f32e7eSjoerg InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
74306f32e7eSjoerg }
74406f32e7eSjoerg
CreateGCStatepointInvoke(uint64_t ID,uint32_t NumPatchBytes,Value * ActualInvokee,BasicBlock * NormalDest,BasicBlock * UnwindDest,ArrayRef<Use> InvokeArgs,Optional<ArrayRef<Value * >> DeoptArgs,ArrayRef<Value * > GCArgs,const Twine & Name)74506f32e7eSjoerg InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
74606f32e7eSjoerg uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
74706f32e7eSjoerg BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
748*da58b97aSjoerg Optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
74906f32e7eSjoerg return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
75006f32e7eSjoerg this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
75106f32e7eSjoerg uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
75206f32e7eSjoerg Name);
75306f32e7eSjoerg }
75406f32e7eSjoerg
CreateGCResult(Instruction * Statepoint,Type * ResultType,const Twine & Name)75506f32e7eSjoerg CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
75606f32e7eSjoerg Type *ResultType,
75706f32e7eSjoerg const Twine &Name) {
75806f32e7eSjoerg Intrinsic::ID ID = Intrinsic::experimental_gc_result;
75906f32e7eSjoerg Module *M = BB->getParent()->getParent();
76006f32e7eSjoerg Type *Types[] = {ResultType};
76106f32e7eSjoerg Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
76206f32e7eSjoerg
76306f32e7eSjoerg Value *Args[] = {Statepoint};
76406f32e7eSjoerg return createCallHelper(FnGCResult, Args, this, Name);
76506f32e7eSjoerg }
76606f32e7eSjoerg
CreateGCRelocate(Instruction * Statepoint,int BaseOffset,int DerivedOffset,Type * ResultType,const Twine & Name)76706f32e7eSjoerg CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
76806f32e7eSjoerg int BaseOffset,
76906f32e7eSjoerg int DerivedOffset,
77006f32e7eSjoerg Type *ResultType,
77106f32e7eSjoerg const Twine &Name) {
77206f32e7eSjoerg Module *M = BB->getParent()->getParent();
77306f32e7eSjoerg Type *Types[] = {ResultType};
77406f32e7eSjoerg Function *FnGCRelocate =
77506f32e7eSjoerg Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
77606f32e7eSjoerg
77706f32e7eSjoerg Value *Args[] = {Statepoint,
77806f32e7eSjoerg getInt32(BaseOffset),
77906f32e7eSjoerg getInt32(DerivedOffset)};
78006f32e7eSjoerg return createCallHelper(FnGCRelocate, Args, this, Name);
78106f32e7eSjoerg }
78206f32e7eSjoerg
CreateUnaryIntrinsic(Intrinsic::ID ID,Value * V,Instruction * FMFSource,const Twine & Name)78306f32e7eSjoerg CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
78406f32e7eSjoerg Instruction *FMFSource,
78506f32e7eSjoerg const Twine &Name) {
78606f32e7eSjoerg Module *M = BB->getModule();
78706f32e7eSjoerg Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
78806f32e7eSjoerg return createCallHelper(Fn, {V}, this, Name, FMFSource);
78906f32e7eSjoerg }
79006f32e7eSjoerg
CreateBinaryIntrinsic(Intrinsic::ID ID,Value * LHS,Value * RHS,Instruction * FMFSource,const Twine & Name)79106f32e7eSjoerg CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
79206f32e7eSjoerg Value *RHS,
79306f32e7eSjoerg Instruction *FMFSource,
79406f32e7eSjoerg const Twine &Name) {
79506f32e7eSjoerg Module *M = BB->getModule();
79606f32e7eSjoerg Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
79706f32e7eSjoerg return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
79806f32e7eSjoerg }
79906f32e7eSjoerg
CreateIntrinsic(Intrinsic::ID ID,ArrayRef<Type * > Types,ArrayRef<Value * > Args,Instruction * FMFSource,const Twine & Name)80006f32e7eSjoerg CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
80106f32e7eSjoerg ArrayRef<Type *> Types,
80206f32e7eSjoerg ArrayRef<Value *> Args,
80306f32e7eSjoerg Instruction *FMFSource,
80406f32e7eSjoerg const Twine &Name) {
80506f32e7eSjoerg Module *M = BB->getModule();
80606f32e7eSjoerg Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
80706f32e7eSjoerg return createCallHelper(Fn, Args, this, Name, FMFSource);
80806f32e7eSjoerg }
809*da58b97aSjoerg
CreateConstrainedFPBinOp(Intrinsic::ID ID,Value * L,Value * R,Instruction * FMFSource,const Twine & Name,MDNode * FPMathTag,Optional<RoundingMode> Rounding,Optional<fp::ExceptionBehavior> Except)810*da58b97aSjoerg CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
811*da58b97aSjoerg Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
812*da58b97aSjoerg const Twine &Name, MDNode *FPMathTag,
813*da58b97aSjoerg Optional<RoundingMode> Rounding,
814*da58b97aSjoerg Optional<fp::ExceptionBehavior> Except) {
815*da58b97aSjoerg Value *RoundingV = getConstrainedFPRounding(Rounding);
816*da58b97aSjoerg Value *ExceptV = getConstrainedFPExcept(Except);
817*da58b97aSjoerg
818*da58b97aSjoerg FastMathFlags UseFMF = FMF;
819*da58b97aSjoerg if (FMFSource)
820*da58b97aSjoerg UseFMF = FMFSource->getFastMathFlags();
821*da58b97aSjoerg
822*da58b97aSjoerg CallInst *C = CreateIntrinsic(ID, {L->getType()},
823*da58b97aSjoerg {L, R, RoundingV, ExceptV}, nullptr, Name);
824*da58b97aSjoerg setConstrainedFPCallAttr(C);
825*da58b97aSjoerg setFPAttrs(C, FPMathTag, UseFMF);
826*da58b97aSjoerg return C;
827*da58b97aSjoerg }
828*da58b97aSjoerg
CreateNAryOp(unsigned Opc,ArrayRef<Value * > Ops,const Twine & Name,MDNode * FPMathTag)829*da58b97aSjoerg Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
830*da58b97aSjoerg const Twine &Name, MDNode *FPMathTag) {
831*da58b97aSjoerg if (Instruction::isBinaryOp(Opc)) {
832*da58b97aSjoerg assert(Ops.size() == 2 && "Invalid number of operands!");
833*da58b97aSjoerg return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
834*da58b97aSjoerg Ops[0], Ops[1], Name, FPMathTag);
835*da58b97aSjoerg }
836*da58b97aSjoerg if (Instruction::isUnaryOp(Opc)) {
837*da58b97aSjoerg assert(Ops.size() == 1 && "Invalid number of operands!");
838*da58b97aSjoerg return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
839*da58b97aSjoerg Ops[0], Name, FPMathTag);
840*da58b97aSjoerg }
841*da58b97aSjoerg llvm_unreachable("Unexpected opcode!");
842*da58b97aSjoerg }
843*da58b97aSjoerg
CreateConstrainedFPCast(Intrinsic::ID ID,Value * V,Type * DestTy,Instruction * FMFSource,const Twine & Name,MDNode * FPMathTag,Optional<RoundingMode> Rounding,Optional<fp::ExceptionBehavior> Except)844*da58b97aSjoerg CallInst *IRBuilderBase::CreateConstrainedFPCast(
845*da58b97aSjoerg Intrinsic::ID ID, Value *V, Type *DestTy,
846*da58b97aSjoerg Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
847*da58b97aSjoerg Optional<RoundingMode> Rounding,
848*da58b97aSjoerg Optional<fp::ExceptionBehavior> Except) {
849*da58b97aSjoerg Value *ExceptV = getConstrainedFPExcept(Except);
850*da58b97aSjoerg
851*da58b97aSjoerg FastMathFlags UseFMF = FMF;
852*da58b97aSjoerg if (FMFSource)
853*da58b97aSjoerg UseFMF = FMFSource->getFastMathFlags();
854*da58b97aSjoerg
855*da58b97aSjoerg CallInst *C;
856*da58b97aSjoerg bool HasRoundingMD = false;
857*da58b97aSjoerg switch (ID) {
858*da58b97aSjoerg default:
859*da58b97aSjoerg break;
860*da58b97aSjoerg #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
861*da58b97aSjoerg case Intrinsic::INTRINSIC: \
862*da58b97aSjoerg HasRoundingMD = ROUND_MODE; \
863*da58b97aSjoerg break;
864*da58b97aSjoerg #include "llvm/IR/ConstrainedOps.def"
865*da58b97aSjoerg }
866*da58b97aSjoerg if (HasRoundingMD) {
867*da58b97aSjoerg Value *RoundingV = getConstrainedFPRounding(Rounding);
868*da58b97aSjoerg C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
869*da58b97aSjoerg nullptr, Name);
870*da58b97aSjoerg } else
871*da58b97aSjoerg C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
872*da58b97aSjoerg Name);
873*da58b97aSjoerg
874*da58b97aSjoerg setConstrainedFPCallAttr(C);
875*da58b97aSjoerg
876*da58b97aSjoerg if (isa<FPMathOperator>(C))
877*da58b97aSjoerg setFPAttrs(C, FPMathTag, UseFMF);
878*da58b97aSjoerg return C;
879*da58b97aSjoerg }
880*da58b97aSjoerg
CreateFCmpHelper(CmpInst::Predicate P,Value * LHS,Value * RHS,const Twine & Name,MDNode * FPMathTag,bool IsSignaling)881*da58b97aSjoerg Value *IRBuilderBase::CreateFCmpHelper(
882*da58b97aSjoerg CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
883*da58b97aSjoerg MDNode *FPMathTag, bool IsSignaling) {
884*da58b97aSjoerg if (IsFPConstrained) {
885*da58b97aSjoerg auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
886*da58b97aSjoerg : Intrinsic::experimental_constrained_fcmp;
887*da58b97aSjoerg return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
888*da58b97aSjoerg }
889*da58b97aSjoerg
890*da58b97aSjoerg if (auto *LC = dyn_cast<Constant>(LHS))
891*da58b97aSjoerg if (auto *RC = dyn_cast<Constant>(RHS))
892*da58b97aSjoerg return Insert(Folder.CreateFCmp(P, LC, RC), Name);
893*da58b97aSjoerg return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
894*da58b97aSjoerg }
895*da58b97aSjoerg
CreateConstrainedFPCmp(Intrinsic::ID ID,CmpInst::Predicate P,Value * L,Value * R,const Twine & Name,Optional<fp::ExceptionBehavior> Except)896*da58b97aSjoerg CallInst *IRBuilderBase::CreateConstrainedFPCmp(
897*da58b97aSjoerg Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
898*da58b97aSjoerg const Twine &Name, Optional<fp::ExceptionBehavior> Except) {
899*da58b97aSjoerg Value *PredicateV = getConstrainedFPPredicate(P);
900*da58b97aSjoerg Value *ExceptV = getConstrainedFPExcept(Except);
901*da58b97aSjoerg
902*da58b97aSjoerg CallInst *C = CreateIntrinsic(ID, {L->getType()},
903*da58b97aSjoerg {L, R, PredicateV, ExceptV}, nullptr, Name);
904*da58b97aSjoerg setConstrainedFPCallAttr(C);
905*da58b97aSjoerg return C;
906*da58b97aSjoerg }
907*da58b97aSjoerg
CreateConstrainedFPCall(Function * Callee,ArrayRef<Value * > Args,const Twine & Name,Optional<RoundingMode> Rounding,Optional<fp::ExceptionBehavior> Except)908*da58b97aSjoerg CallInst *IRBuilderBase::CreateConstrainedFPCall(
909*da58b97aSjoerg Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
910*da58b97aSjoerg Optional<RoundingMode> Rounding,
911*da58b97aSjoerg Optional<fp::ExceptionBehavior> Except) {
912*da58b97aSjoerg llvm::SmallVector<Value *, 6> UseArgs;
913*da58b97aSjoerg
914*da58b97aSjoerg append_range(UseArgs, Args);
915*da58b97aSjoerg bool HasRoundingMD = false;
916*da58b97aSjoerg switch (Callee->getIntrinsicID()) {
917*da58b97aSjoerg default:
918*da58b97aSjoerg break;
919*da58b97aSjoerg #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
920*da58b97aSjoerg case Intrinsic::INTRINSIC: \
921*da58b97aSjoerg HasRoundingMD = ROUND_MODE; \
922*da58b97aSjoerg break;
923*da58b97aSjoerg #include "llvm/IR/ConstrainedOps.def"
924*da58b97aSjoerg }
925*da58b97aSjoerg if (HasRoundingMD)
926*da58b97aSjoerg UseArgs.push_back(getConstrainedFPRounding(Rounding));
927*da58b97aSjoerg UseArgs.push_back(getConstrainedFPExcept(Except));
928*da58b97aSjoerg
929*da58b97aSjoerg CallInst *C = CreateCall(Callee, UseArgs, Name);
930*da58b97aSjoerg setConstrainedFPCallAttr(C);
931*da58b97aSjoerg return C;
932*da58b97aSjoerg }
933*da58b97aSjoerg
CreateSelect(Value * C,Value * True,Value * False,const Twine & Name,Instruction * MDFrom)934*da58b97aSjoerg Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
935*da58b97aSjoerg const Twine &Name, Instruction *MDFrom) {
936*da58b97aSjoerg if (auto *CC = dyn_cast<Constant>(C))
937*da58b97aSjoerg if (auto *TC = dyn_cast<Constant>(True))
938*da58b97aSjoerg if (auto *FC = dyn_cast<Constant>(False))
939*da58b97aSjoerg return Insert(Folder.CreateSelect(CC, TC, FC), Name);
940*da58b97aSjoerg
941*da58b97aSjoerg SelectInst *Sel = SelectInst::Create(C, True, False);
942*da58b97aSjoerg if (MDFrom) {
943*da58b97aSjoerg MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
944*da58b97aSjoerg MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
945*da58b97aSjoerg Sel = addBranchMetadata(Sel, Prof, Unpred);
946*da58b97aSjoerg }
947*da58b97aSjoerg if (isa<FPMathOperator>(Sel))
948*da58b97aSjoerg setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
949*da58b97aSjoerg return Insert(Sel, Name);
950*da58b97aSjoerg }
951*da58b97aSjoerg
CreatePtrDiff(Value * LHS,Value * RHS,const Twine & Name)952*da58b97aSjoerg Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS,
953*da58b97aSjoerg const Twine &Name) {
954*da58b97aSjoerg assert(LHS->getType() == RHS->getType() &&
955*da58b97aSjoerg "Pointer subtraction operand types must match!");
956*da58b97aSjoerg auto *ArgType = cast<PointerType>(LHS->getType());
957*da58b97aSjoerg Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
958*da58b97aSjoerg Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
959*da58b97aSjoerg Value *Difference = CreateSub(LHS_int, RHS_int);
960*da58b97aSjoerg return CreateExactSDiv(Difference,
961*da58b97aSjoerg ConstantExpr::getSizeOf(ArgType->getElementType()),
962*da58b97aSjoerg Name);
963*da58b97aSjoerg }
964*da58b97aSjoerg
CreateLaunderInvariantGroup(Value * Ptr)965*da58b97aSjoerg Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
966*da58b97aSjoerg assert(isa<PointerType>(Ptr->getType()) &&
967*da58b97aSjoerg "launder.invariant.group only applies to pointers.");
968*da58b97aSjoerg // FIXME: we could potentially avoid casts to/from i8*.
969*da58b97aSjoerg auto *PtrType = Ptr->getType();
970*da58b97aSjoerg auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
971*da58b97aSjoerg if (PtrType != Int8PtrTy)
972*da58b97aSjoerg Ptr = CreateBitCast(Ptr, Int8PtrTy);
973*da58b97aSjoerg Module *M = BB->getParent()->getParent();
974*da58b97aSjoerg Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
975*da58b97aSjoerg M, Intrinsic::launder_invariant_group, {Int8PtrTy});
976*da58b97aSjoerg
977*da58b97aSjoerg assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&
978*da58b97aSjoerg FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
979*da58b97aSjoerg Int8PtrTy &&
980*da58b97aSjoerg "LaunderInvariantGroup should take and return the same type");
981*da58b97aSjoerg
982*da58b97aSjoerg CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr});
983*da58b97aSjoerg
984*da58b97aSjoerg if (PtrType != Int8PtrTy)
985*da58b97aSjoerg return CreateBitCast(Fn, PtrType);
986*da58b97aSjoerg return Fn;
987*da58b97aSjoerg }
988*da58b97aSjoerg
CreateStripInvariantGroup(Value * Ptr)989*da58b97aSjoerg Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
990*da58b97aSjoerg assert(isa<PointerType>(Ptr->getType()) &&
991*da58b97aSjoerg "strip.invariant.group only applies to pointers.");
992*da58b97aSjoerg
993*da58b97aSjoerg // FIXME: we could potentially avoid casts to/from i8*.
994*da58b97aSjoerg auto *PtrType = Ptr->getType();
995*da58b97aSjoerg auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace());
996*da58b97aSjoerg if (PtrType != Int8PtrTy)
997*da58b97aSjoerg Ptr = CreateBitCast(Ptr, Int8PtrTy);
998*da58b97aSjoerg Module *M = BB->getParent()->getParent();
999*da58b97aSjoerg Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
1000*da58b97aSjoerg M, Intrinsic::strip_invariant_group, {Int8PtrTy});
1001*da58b97aSjoerg
1002*da58b97aSjoerg assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&
1003*da58b97aSjoerg FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
1004*da58b97aSjoerg Int8PtrTy &&
1005*da58b97aSjoerg "StripInvariantGroup should take and return the same type");
1006*da58b97aSjoerg
1007*da58b97aSjoerg CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr});
1008*da58b97aSjoerg
1009*da58b97aSjoerg if (PtrType != Int8PtrTy)
1010*da58b97aSjoerg return CreateBitCast(Fn, PtrType);
1011*da58b97aSjoerg return Fn;
1012*da58b97aSjoerg }
1013*da58b97aSjoerg
CreateVectorReverse(Value * V,const Twine & Name)1014*da58b97aSjoerg Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1015*da58b97aSjoerg auto *Ty = cast<VectorType>(V->getType());
1016*da58b97aSjoerg if (isa<ScalableVectorType>(Ty)) {
1017*da58b97aSjoerg Module *M = BB->getParent()->getParent();
1018*da58b97aSjoerg Function *F = Intrinsic::getDeclaration(
1019*da58b97aSjoerg M, Intrinsic::experimental_vector_reverse, Ty);
1020*da58b97aSjoerg return Insert(CallInst::Create(F, V), Name);
1021*da58b97aSjoerg }
1022*da58b97aSjoerg // Keep the original behaviour for fixed vector
1023*da58b97aSjoerg SmallVector<int, 8> ShuffleMask;
1024*da58b97aSjoerg int NumElts = Ty->getElementCount().getKnownMinValue();
1025*da58b97aSjoerg for (int i = 0; i < NumElts; ++i)
1026*da58b97aSjoerg ShuffleMask.push_back(NumElts - i - 1);
1027*da58b97aSjoerg return CreateShuffleVector(V, ShuffleMask, Name);
1028*da58b97aSjoerg }
1029*da58b97aSjoerg
CreateVectorSplice(Value * V1,Value * V2,int64_t Imm,const Twine & Name)1030*da58b97aSjoerg Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
1031*da58b97aSjoerg const Twine &Name) {
1032*da58b97aSjoerg assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1033*da58b97aSjoerg assert(V1->getType() == V2->getType() &&
1034*da58b97aSjoerg "Splice expects matching operand types!");
1035*da58b97aSjoerg
1036*da58b97aSjoerg if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1037*da58b97aSjoerg Module *M = BB->getParent()->getParent();
1038*da58b97aSjoerg Function *F = Intrinsic::getDeclaration(
1039*da58b97aSjoerg M, Intrinsic::experimental_vector_splice, VTy);
1040*da58b97aSjoerg
1041*da58b97aSjoerg Value *Ops[] = {V1, V2, getInt32(Imm)};
1042*da58b97aSjoerg return Insert(CallInst::Create(F, Ops), Name);
1043*da58b97aSjoerg }
1044*da58b97aSjoerg
1045*da58b97aSjoerg unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1046*da58b97aSjoerg assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1047*da58b97aSjoerg "Invalid immediate for vector splice!");
1048*da58b97aSjoerg
1049*da58b97aSjoerg // Keep the original behaviour for fixed vector
1050*da58b97aSjoerg unsigned Idx = (NumElts + Imm) % NumElts;
1051*da58b97aSjoerg SmallVector<int, 8> Mask;
1052*da58b97aSjoerg for (unsigned I = 0; I < NumElts; ++I)
1053*da58b97aSjoerg Mask.push_back(Idx + I);
1054*da58b97aSjoerg
1055*da58b97aSjoerg return CreateShuffleVector(V1, V2, Mask);
1056*da58b97aSjoerg }
1057*da58b97aSjoerg
CreateVectorSplat(unsigned NumElts,Value * V,const Twine & Name)1058*da58b97aSjoerg Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
1059*da58b97aSjoerg const Twine &Name) {
1060*da58b97aSjoerg auto EC = ElementCount::getFixed(NumElts);
1061*da58b97aSjoerg return CreateVectorSplat(EC, V, Name);
1062*da58b97aSjoerg }
1063*da58b97aSjoerg
CreateVectorSplat(ElementCount EC,Value * V,const Twine & Name)1064*da58b97aSjoerg Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1065*da58b97aSjoerg const Twine &Name) {
1066*da58b97aSjoerg assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1067*da58b97aSjoerg
1068*da58b97aSjoerg // First insert it into a poison vector so we can shuffle it.
1069*da58b97aSjoerg Type *I32Ty = getInt32Ty();
1070*da58b97aSjoerg Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1071*da58b97aSjoerg V = CreateInsertElement(Poison, V, ConstantInt::get(I32Ty, 0),
1072*da58b97aSjoerg Name + ".splatinsert");
1073*da58b97aSjoerg
1074*da58b97aSjoerg // Shuffle the value across the desired number of elements.
1075*da58b97aSjoerg SmallVector<int, 16> Zeros;
1076*da58b97aSjoerg Zeros.resize(EC.getKnownMinValue());
1077*da58b97aSjoerg return CreateShuffleVector(V, Zeros, Name + ".splat");
1078*da58b97aSjoerg }
1079*da58b97aSjoerg
CreateExtractInteger(const DataLayout & DL,Value * From,IntegerType * ExtractedTy,uint64_t Offset,const Twine & Name)1080*da58b97aSjoerg Value *IRBuilderBase::CreateExtractInteger(
1081*da58b97aSjoerg const DataLayout &DL, Value *From, IntegerType *ExtractedTy,
1082*da58b97aSjoerg uint64_t Offset, const Twine &Name) {
1083*da58b97aSjoerg auto *IntTy = cast<IntegerType>(From->getType());
1084*da58b97aSjoerg assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=
1085*da58b97aSjoerg DL.getTypeStoreSize(IntTy) &&
1086*da58b97aSjoerg "Element extends past full value");
1087*da58b97aSjoerg uint64_t ShAmt = 8 * Offset;
1088*da58b97aSjoerg Value *V = From;
1089*da58b97aSjoerg if (DL.isBigEndian())
1090*da58b97aSjoerg ShAmt = 8 * (DL.getTypeStoreSize(IntTy) -
1091*da58b97aSjoerg DL.getTypeStoreSize(ExtractedTy) - Offset);
1092*da58b97aSjoerg if (ShAmt) {
1093*da58b97aSjoerg V = CreateLShr(V, ShAmt, Name + ".shift");
1094*da58b97aSjoerg }
1095*da58b97aSjoerg assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&
1096*da58b97aSjoerg "Cannot extract to a larger integer!");
1097*da58b97aSjoerg if (ExtractedTy != IntTy) {
1098*da58b97aSjoerg V = CreateTrunc(V, ExtractedTy, Name + ".trunc");
1099*da58b97aSjoerg }
1100*da58b97aSjoerg return V;
1101*da58b97aSjoerg }
1102*da58b97aSjoerg
CreatePreserveArrayAccessIndex(Type * ElTy,Value * Base,unsigned Dimension,unsigned LastIndex,MDNode * DbgInfo)1103*da58b97aSjoerg Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
1104*da58b97aSjoerg Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
1105*da58b97aSjoerg MDNode *DbgInfo) {
1106*da58b97aSjoerg assert(isa<PointerType>(Base->getType()) &&
1107*da58b97aSjoerg "Invalid Base ptr type for preserve.array.access.index.");
1108*da58b97aSjoerg auto *BaseType = Base->getType();
1109*da58b97aSjoerg
1110*da58b97aSjoerg Value *LastIndexV = getInt32(LastIndex);
1111*da58b97aSjoerg Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1112*da58b97aSjoerg SmallVector<Value *, 4> IdxList(Dimension, Zero);
1113*da58b97aSjoerg IdxList.push_back(LastIndexV);
1114*da58b97aSjoerg
1115*da58b97aSjoerg Type *ResultType =
1116*da58b97aSjoerg GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList);
1117*da58b97aSjoerg
1118*da58b97aSjoerg Module *M = BB->getParent()->getParent();
1119*da58b97aSjoerg Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
1120*da58b97aSjoerg M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
1121*da58b97aSjoerg
1122*da58b97aSjoerg Value *DimV = getInt32(Dimension);
1123*da58b97aSjoerg CallInst *Fn =
1124*da58b97aSjoerg CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1125*da58b97aSjoerg if (DbgInfo)
1126*da58b97aSjoerg Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1127*da58b97aSjoerg
1128*da58b97aSjoerg return Fn;
1129*da58b97aSjoerg }
1130*da58b97aSjoerg
CreatePreserveUnionAccessIndex(Value * Base,unsigned FieldIndex,MDNode * DbgInfo)1131*da58b97aSjoerg Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
1132*da58b97aSjoerg Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
1133*da58b97aSjoerg assert(isa<PointerType>(Base->getType()) &&
1134*da58b97aSjoerg "Invalid Base ptr type for preserve.union.access.index.");
1135*da58b97aSjoerg auto *BaseType = Base->getType();
1136*da58b97aSjoerg
1137*da58b97aSjoerg Module *M = BB->getParent()->getParent();
1138*da58b97aSjoerg Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
1139*da58b97aSjoerg M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
1140*da58b97aSjoerg
1141*da58b97aSjoerg Value *DIIndex = getInt32(FieldIndex);
1142*da58b97aSjoerg CallInst *Fn =
1143*da58b97aSjoerg CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
1144*da58b97aSjoerg if (DbgInfo)
1145*da58b97aSjoerg Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1146*da58b97aSjoerg
1147*da58b97aSjoerg return Fn;
1148*da58b97aSjoerg }
1149*da58b97aSjoerg
CreatePreserveStructAccessIndex(Type * ElTy,Value * Base,unsigned Index,unsigned FieldIndex,MDNode * DbgInfo)1150*da58b97aSjoerg Value *IRBuilderBase::CreatePreserveStructAccessIndex(
1151*da58b97aSjoerg Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
1152*da58b97aSjoerg MDNode *DbgInfo) {
1153*da58b97aSjoerg assert(isa<PointerType>(Base->getType()) &&
1154*da58b97aSjoerg "Invalid Base ptr type for preserve.struct.access.index.");
1155*da58b97aSjoerg auto *BaseType = Base->getType();
1156*da58b97aSjoerg
1157*da58b97aSjoerg Value *GEPIndex = getInt32(Index);
1158*da58b97aSjoerg Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1159*da58b97aSjoerg Type *ResultType =
1160*da58b97aSjoerg GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex});
1161*da58b97aSjoerg
1162*da58b97aSjoerg Module *M = BB->getParent()->getParent();
1163*da58b97aSjoerg Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
1164*da58b97aSjoerg M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
1165*da58b97aSjoerg
1166*da58b97aSjoerg Value *DIIndex = getInt32(FieldIndex);
1167*da58b97aSjoerg CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
1168*da58b97aSjoerg {Base, GEPIndex, DIIndex});
1169*da58b97aSjoerg if (DbgInfo)
1170*da58b97aSjoerg Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
1171*da58b97aSjoerg
1172*da58b97aSjoerg return Fn;
1173*da58b97aSjoerg }
1174*da58b97aSjoerg
CreateAlignmentAssumptionHelper(const DataLayout & DL,Value * PtrValue,Value * AlignValue,Value * OffsetValue)1175*da58b97aSjoerg CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1176*da58b97aSjoerg Value *PtrValue,
1177*da58b97aSjoerg Value *AlignValue,
1178*da58b97aSjoerg Value *OffsetValue) {
1179*da58b97aSjoerg SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1180*da58b97aSjoerg if (OffsetValue)
1181*da58b97aSjoerg Vals.push_back(OffsetValue);
1182*da58b97aSjoerg OperandBundleDefT<Value *> AlignOpB("align", Vals);
1183*da58b97aSjoerg return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
1184*da58b97aSjoerg }
1185*da58b97aSjoerg
CreateAlignmentAssumption(const DataLayout & DL,Value * PtrValue,unsigned Alignment,Value * OffsetValue)1186*da58b97aSjoerg CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1187*da58b97aSjoerg Value *PtrValue,
1188*da58b97aSjoerg unsigned Alignment,
1189*da58b97aSjoerg Value *OffsetValue) {
1190*da58b97aSjoerg assert(isa<PointerType>(PtrValue->getType()) &&
1191*da58b97aSjoerg "trying to create an alignment assumption on a non-pointer?");
1192*da58b97aSjoerg assert(Alignment != 0 && "Invalid Alignment");
1193*da58b97aSjoerg auto *PtrTy = cast<PointerType>(PtrValue->getType());
1194*da58b97aSjoerg Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1195*da58b97aSjoerg Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1196*da58b97aSjoerg return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
1197*da58b97aSjoerg }
1198*da58b97aSjoerg
CreateAlignmentAssumption(const DataLayout & DL,Value * PtrValue,Value * Alignment,Value * OffsetValue)1199*da58b97aSjoerg CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1200*da58b97aSjoerg Value *PtrValue,
1201*da58b97aSjoerg Value *Alignment,
1202*da58b97aSjoerg Value *OffsetValue) {
1203*da58b97aSjoerg assert(isa<PointerType>(PtrValue->getType()) &&
1204*da58b97aSjoerg "trying to create an alignment assumption on a non-pointer?");
1205*da58b97aSjoerg return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
1206*da58b97aSjoerg }
1207*da58b97aSjoerg
~IRBuilderDefaultInserter()1208*da58b97aSjoerg IRBuilderDefaultInserter::~IRBuilderDefaultInserter() {}
~IRBuilderCallbackInserter()1209*da58b97aSjoerg IRBuilderCallbackInserter::~IRBuilderCallbackInserter() {}
~IRBuilderFolder()1210*da58b97aSjoerg IRBuilderFolder::~IRBuilderFolder() {}
anchor()1211*da58b97aSjoerg void ConstantFolder::anchor() {}
anchor()1212*da58b97aSjoerg void NoFolder::anchor() {}
1213