1480093f4SDimitry Andric //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// 2480093f4SDimitry Andric // 3480093f4SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4480093f4SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5480093f4SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6480093f4SDimitry Andric // 7480093f4SDimitry Andric //===----------------------------------------------------------------------===// 8480093f4SDimitry Andric // 9480093f4SDimitry Andric // Populates the VFABI attribute with the scalar-to-vector mappings 10480093f4SDimitry Andric // from the TargetLibraryInfo. 11480093f4SDimitry Andric // 12480093f4SDimitry Andric //===----------------------------------------------------------------------===// 13480093f4SDimitry Andric 14480093f4SDimitry Andric #include "llvm/Transforms/Utils/InjectTLIMappings.h" 15480093f4SDimitry Andric #include "llvm/ADT/Statistic.h" 165ffd83dbSDimitry Andric #include "llvm/Analysis/DemandedBits.h" 175ffd83dbSDimitry Andric #include "llvm/Analysis/GlobalsModRef.h" 185ffd83dbSDimitry Andric #include "llvm/Analysis/OptimizationRemarkEmitter.h" 19e8d8bef9SDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h" 20480093f4SDimitry Andric #include "llvm/Analysis/VectorUtils.h" 21480093f4SDimitry Andric #include "llvm/IR/InstIterator.h" 225ffd83dbSDimitry Andric #include "llvm/IR/IntrinsicInst.h" 23480093f4SDimitry Andric #include "llvm/Transforms/Utils.h" 24480093f4SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h" 25480093f4SDimitry Andric 26480093f4SDimitry Andric using namespace llvm; 27480093f4SDimitry Andric 28480093f4SDimitry Andric #define DEBUG_TYPE "inject-tli-mappings" 29480093f4SDimitry Andric 30480093f4SDimitry Andric STATISTIC(NumCallInjected, 31480093f4SDimitry Andric "Number of calls in which the mappings have been injected."); 32480093f4SDimitry Andric 33480093f4SDimitry Andric STATISTIC(NumVFDeclAdded, 34480093f4SDimitry Andric "Number of function declarations that have been added."); 35480093f4SDimitry Andric STATISTIC(NumCompUsedAdded, 36480093f4SDimitry Andric "Number of `@llvm.compiler.used` operands that have been added."); 37480093f4SDimitry Andric 38480093f4SDimitry Andric /// A helper function that adds the vector function declaration that 39480093f4SDimitry Andric /// vectorizes the CallInst CI with a vectorization factor of VF 40480093f4SDimitry Andric /// lanes. The TLI assumes that all parameters and the return type of 41480093f4SDimitry Andric /// CI (other than void) need to be widened to a VectorType of VF 42480093f4SDimitry Andric /// lanes. 43*fe6060f1SDimitry Andric static void addVariantDeclaration(CallInst &CI, const ElementCount &VF, 44480093f4SDimitry Andric const StringRef VFName) { 45480093f4SDimitry Andric Module *M = CI.getModule(); 46480093f4SDimitry Andric 47480093f4SDimitry Andric // Add function declaration. 48480093f4SDimitry Andric Type *RetTy = ToVectorTy(CI.getType(), VF); 49480093f4SDimitry Andric SmallVector<Type *, 4> Tys; 50480093f4SDimitry Andric for (Value *ArgOperand : CI.arg_operands()) 51480093f4SDimitry Andric Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); 52480093f4SDimitry Andric assert(!CI.getFunctionType()->isVarArg() && 53480093f4SDimitry Andric "VarArg functions are not supported."); 54480093f4SDimitry Andric FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); 55480093f4SDimitry Andric Function *VectorF = 56480093f4SDimitry Andric Function::Create(FTy, Function::ExternalLinkage, VFName, M); 57480093f4SDimitry Andric VectorF->copyAttributesFrom(CI.getCalledFunction()); 58480093f4SDimitry Andric ++NumVFDeclAdded; 59480093f4SDimitry Andric LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName 60480093f4SDimitry Andric << "` of type " << *(VectorF->getType()) << "\n"); 61480093f4SDimitry Andric 62480093f4SDimitry Andric // Make function declaration (without a body) "sticky" in the IR by 63480093f4SDimitry Andric // listing it in the @llvm.compiler.used intrinsic. 64480093f4SDimitry Andric assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " 65480093f4SDimitry Andric "only on declarations."); 66480093f4SDimitry Andric appendToCompilerUsed(*M, {VectorF}); 67480093f4SDimitry Andric LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName 68480093f4SDimitry Andric << "` to `@llvm.compiler.used`.\n"); 69480093f4SDimitry Andric ++NumCompUsedAdded; 70480093f4SDimitry Andric } 71480093f4SDimitry Andric 72480093f4SDimitry Andric static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { 73480093f4SDimitry Andric // This is needed to make sure we don't query the TLI for calls to 74480093f4SDimitry Andric // bitcast of function pointers, like `%call = call i32 (i32*, ...) 75480093f4SDimitry Andric // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, 76480093f4SDimitry Andric // as such calls make the `isFunctionVectorizable` raise an 77480093f4SDimitry Andric // exception. 78480093f4SDimitry Andric if (CI.isNoBuiltin() || !CI.getCalledFunction()) 79480093f4SDimitry Andric return; 80480093f4SDimitry Andric 81e8d8bef9SDimitry Andric StringRef ScalarName = CI.getCalledFunction()->getName(); 82e8d8bef9SDimitry Andric 83480093f4SDimitry Andric // Nothing to be done if the TLI thinks the function is not 84480093f4SDimitry Andric // vectorizable. 85480093f4SDimitry Andric if (!TLI.isFunctionVectorizable(ScalarName)) 86480093f4SDimitry Andric return; 87480093f4SDimitry Andric SmallVector<std::string, 8> Mappings; 88480093f4SDimitry Andric VFABI::getVectorVariantNames(CI, Mappings); 89480093f4SDimitry Andric Module *M = CI.getModule(); 90480093f4SDimitry Andric const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(), 91480093f4SDimitry Andric Mappings.end()); 92*fe6060f1SDimitry Andric 93*fe6060f1SDimitry Andric auto AddVariantDecl = [&](const ElementCount &VF) { 945ffd83dbSDimitry Andric const std::string TLIName = 955ffd83dbSDimitry Andric std::string(TLI.getVectorizedFunction(ScalarName, VF)); 96480093f4SDimitry Andric if (!TLIName.empty()) { 975ffd83dbSDimitry Andric std::string MangledName = VFABI::mangleTLIVectorName( 985ffd83dbSDimitry Andric TLIName, ScalarName, CI.getNumArgOperands(), VF); 99480093f4SDimitry Andric if (!OriginalSetOfMappings.count(MangledName)) { 100480093f4SDimitry Andric Mappings.push_back(MangledName); 101480093f4SDimitry Andric ++NumCallInjected; 102480093f4SDimitry Andric } 103480093f4SDimitry Andric Function *VariantF = M->getFunction(TLIName); 104480093f4SDimitry Andric if (!VariantF) 105480093f4SDimitry Andric addVariantDeclaration(CI, VF, TLIName); 106480093f4SDimitry Andric } 107*fe6060f1SDimitry Andric }; 108*fe6060f1SDimitry Andric 109*fe6060f1SDimitry Andric // All VFs in the TLI are powers of 2. 110*fe6060f1SDimitry Andric ElementCount WidestFixedVF, WidestScalableVF; 111*fe6060f1SDimitry Andric TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF); 112*fe6060f1SDimitry Andric 113*fe6060f1SDimitry Andric for (ElementCount VF = ElementCount::getFixed(2); 114*fe6060f1SDimitry Andric ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2) 115*fe6060f1SDimitry Andric AddVariantDecl(VF); 116*fe6060f1SDimitry Andric 117*fe6060f1SDimitry Andric // TODO: Add scalable variants once we're able to test them. 118*fe6060f1SDimitry Andric assert(WidestScalableVF.isZero() && 119*fe6060f1SDimitry Andric "Scalable vector mappings not yet supported"); 120480093f4SDimitry Andric 121480093f4SDimitry Andric VFABI::setVectorVariantNames(&CI, Mappings); 122480093f4SDimitry Andric } 123480093f4SDimitry Andric 124480093f4SDimitry Andric static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { 125480093f4SDimitry Andric for (auto &I : instructions(F)) 126480093f4SDimitry Andric if (auto CI = dyn_cast<CallInst>(&I)) 127480093f4SDimitry Andric addMappingsFromTLI(TLI, *CI); 128480093f4SDimitry Andric // Even if the pass adds IR attributes, the analyses are preserved. 129480093f4SDimitry Andric return false; 130480093f4SDimitry Andric } 131480093f4SDimitry Andric 132480093f4SDimitry Andric //////////////////////////////////////////////////////////////////////////////// 133480093f4SDimitry Andric // New pass manager implementation. 134480093f4SDimitry Andric //////////////////////////////////////////////////////////////////////////////// 135480093f4SDimitry Andric PreservedAnalyses InjectTLIMappings::run(Function &F, 136480093f4SDimitry Andric FunctionAnalysisManager &AM) { 137480093f4SDimitry Andric const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F); 138480093f4SDimitry Andric runImpl(TLI, F); 139480093f4SDimitry Andric // Even if the pass adds IR attributes, the analyses are preserved. 140480093f4SDimitry Andric return PreservedAnalyses::all(); 141480093f4SDimitry Andric } 142480093f4SDimitry Andric 143480093f4SDimitry Andric //////////////////////////////////////////////////////////////////////////////// 144480093f4SDimitry Andric // Legacy PM Implementation. 145480093f4SDimitry Andric //////////////////////////////////////////////////////////////////////////////// 146480093f4SDimitry Andric bool InjectTLIMappingsLegacy::runOnFunction(Function &F) { 147480093f4SDimitry Andric const TargetLibraryInfo &TLI = 148480093f4SDimitry Andric getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F); 149480093f4SDimitry Andric return runImpl(TLI, F); 150480093f4SDimitry Andric } 151480093f4SDimitry Andric 152480093f4SDimitry Andric void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { 153480093f4SDimitry Andric AU.setPreservesCFG(); 154480093f4SDimitry Andric AU.addRequired<TargetLibraryInfoWrapperPass>(); 155480093f4SDimitry Andric AU.addPreserved<TargetLibraryInfoWrapperPass>(); 1565ffd83dbSDimitry Andric AU.addPreserved<ScalarEvolutionWrapperPass>(); 1575ffd83dbSDimitry Andric AU.addPreserved<AAResultsWrapperPass>(); 1585ffd83dbSDimitry Andric AU.addPreserved<LoopAccessLegacyAnalysis>(); 1595ffd83dbSDimitry Andric AU.addPreserved<DemandedBitsWrapperPass>(); 1605ffd83dbSDimitry Andric AU.addPreserved<OptimizationRemarkEmitterWrapperPass>(); 1615ffd83dbSDimitry Andric AU.addPreserved<GlobalsAAWrapperPass>(); 162480093f4SDimitry Andric } 163480093f4SDimitry Andric 164480093f4SDimitry Andric //////////////////////////////////////////////////////////////////////////////// 165480093f4SDimitry Andric // Legacy Pass manager initialization 166480093f4SDimitry Andric //////////////////////////////////////////////////////////////////////////////// 167480093f4SDimitry Andric char InjectTLIMappingsLegacy::ID = 0; 168480093f4SDimitry Andric 169480093f4SDimitry Andric INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, 170480093f4SDimitry Andric "Inject TLI Mappings", false, false) 171480093f4SDimitry Andric INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 172480093f4SDimitry Andric INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", 173480093f4SDimitry Andric false, false) 174480093f4SDimitry Andric 175480093f4SDimitry Andric FunctionPass *llvm::createInjectTLIMappingsLegacyPass() { 176480093f4SDimitry Andric return new InjectTLIMappingsLegacy(); 177480093f4SDimitry Andric } 178