1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection  ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/DemandedBits.h"
17 #include "llvm/Analysis/GlobalsModRef.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/Analysis/VectorUtils.h"
20 #include "llvm/IR/InstIterator.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/Transforms/Utils.h"
23 #include "llvm/Transforms/Utils/ModuleUtils.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "inject-tli-mappings"
28 
29 STATISTIC(NumCallInjected,
30           "Number of calls in which the mappings have been injected.");
31 
32 STATISTIC(NumVFDeclAdded,
33           "Number of function declarations that have been added.");
34 STATISTIC(NumCompUsedAdded,
35           "Number of `@llvm.compiler.used` operands that have been added.");
36 
37 /// A helper function that adds the vector function declaration that
38 /// vectorizes the CallInst CI with a vectorization factor of VF
39 /// lanes. The TLI assumes that all parameters and the return type of
40 /// CI (other than void) need to be widened to a VectorType of VF
41 /// lanes.
42 static void addVariantDeclaration(CallInst &CI, const unsigned VF,
43                                   const StringRef VFName) {
44   Module *M = CI.getModule();
45 
46   // Add function declaration.
47   Type *RetTy = ToVectorTy(CI.getType(), VF);
48   SmallVector<Type *, 4> Tys;
49   for (Value *ArgOperand : CI.arg_operands())
50     Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
51   assert(!CI.getFunctionType()->isVarArg() &&
52          "VarArg functions are not supported.");
53   FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
54   Function *VectorF =
55       Function::Create(FTy, Function::ExternalLinkage, VFName, M);
56   VectorF->copyAttributesFrom(CI.getCalledFunction());
57   ++NumVFDeclAdded;
58   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
59                     << "` of type " << *(VectorF->getType()) << "\n");
60 
61   // Make function declaration (without a body) "sticky" in the IR by
62   // listing it in the @llvm.compiler.used intrinsic.
63   assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
64                              "only on declarations.");
65   appendToCompilerUsed(*M, {VectorF});
66   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
67                     << "` to `@llvm.compiler.used`.\n");
68   ++NumCompUsedAdded;
69 }
70 
71 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
72   // This is needed to make sure we don't query the TLI for calls to
73   // bitcast of function pointers, like `%call = call i32 (i32*, ...)
74   // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
75   // as such calls make the `isFunctionVectorizable` raise an
76   // exception.
77   if (CI.isNoBuiltin() || !CI.getCalledFunction())
78     return;
79 
80   const std::string ScalarName = std::string(CI.getCalledFunction()->getName());
81   // Nothing to be done if the TLI thinks the function is not
82   // vectorizable.
83   if (!TLI.isFunctionVectorizable(ScalarName))
84     return;
85   SmallVector<std::string, 8> Mappings;
86   VFABI::getVectorVariantNames(CI, Mappings);
87   Module *M = CI.getModule();
88   const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
89                                                    Mappings.end());
90   //  All VFs in the TLI are powers of 2.
91   for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
92        VF *= 2) {
93     const std::string TLIName =
94         std::string(TLI.getVectorizedFunction(ScalarName, VF));
95     if (!TLIName.empty()) {
96       std::string MangledName = VFABI::mangleTLIVectorName(
97           TLIName, ScalarName, CI.getNumArgOperands(), VF);
98       if (!OriginalSetOfMappings.count(MangledName)) {
99         Mappings.push_back(MangledName);
100         ++NumCallInjected;
101       }
102       Function *VariantF = M->getFunction(TLIName);
103       if (!VariantF)
104         addVariantDeclaration(CI, VF, TLIName);
105     }
106   }
107 
108   VFABI::setVectorVariantNames(&CI, Mappings);
109 }
110 
111 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
112   for (auto &I : instructions(F))
113     if (auto CI = dyn_cast<CallInst>(&I))
114       addMappingsFromTLI(TLI, *CI);
115   // Even if the pass adds IR attributes, the analyses are preserved.
116   return false;
117 }
118 
119 ////////////////////////////////////////////////////////////////////////////////
120 // New pass manager implementation.
121 ////////////////////////////////////////////////////////////////////////////////
122 PreservedAnalyses InjectTLIMappings::run(Function &F,
123                                          FunctionAnalysisManager &AM) {
124   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
125   runImpl(TLI, F);
126   // Even if the pass adds IR attributes, the analyses are preserved.
127   return PreservedAnalyses::all();
128 }
129 
130 ////////////////////////////////////////////////////////////////////////////////
131 // Legacy PM Implementation.
132 ////////////////////////////////////////////////////////////////////////////////
133 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
134   const TargetLibraryInfo &TLI =
135       getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
136   return runImpl(TLI, F);
137 }
138 
139 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
140   AU.setPreservesCFG();
141   AU.addRequired<TargetLibraryInfoWrapperPass>();
142   AU.addPreserved<TargetLibraryInfoWrapperPass>();
143   AU.addPreserved<ScalarEvolutionWrapperPass>();
144   AU.addPreserved<AAResultsWrapperPass>();
145   AU.addPreserved<LoopAccessLegacyAnalysis>();
146   AU.addPreserved<DemandedBitsWrapperPass>();
147   AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
148   AU.addPreserved<GlobalsAAWrapperPass>();
149 }
150 
151 ////////////////////////////////////////////////////////////////////////////////
152 // Legacy Pass manager initialization
153 ////////////////////////////////////////////////////////////////////////////////
154 char InjectTLIMappingsLegacy::ID = 0;
155 
156 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
157                       "Inject TLI Mappings", false, false)
158 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
159 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
160                     false, false)
161 
162 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
163   return new InjectTLIMappingsLegacy();
164 }
165