1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection  ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/DemandedBits.h"
17 #include "llvm/Analysis/GlobalsModRef.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/Analysis/TargetLibraryInfo.h"
20 #include "llvm/Analysis/VectorUtils.h"
21 #include "llvm/IR/InstIterator.h"
22 #include "llvm/Transforms/Utils/ModuleUtils.h"
23 
24 using namespace llvm;
25 
26 #define DEBUG_TYPE "inject-tli-mappings"
27 
28 STATISTIC(NumCallInjected,
29           "Number of calls in which the mappings have been injected.");
30 
31 STATISTIC(NumVFDeclAdded,
32           "Number of function declarations that have been added.");
33 STATISTIC(NumCompUsedAdded,
34           "Number of `@llvm.compiler.used` operands that have been added.");
35 
36 /// A helper function that adds the vector function declaration that
37 /// vectorizes the CallInst CI with a vectorization factor of VF
38 /// lanes. The TLI assumes that all parameters and the return type of
39 /// CI (other than void) need to be widened to a VectorType of VF
40 /// lanes.
41 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
42                                   bool Predicate, const StringRef VFName) {
43   Module *M = CI.getModule();
44 
45   // Add function declaration.
46   Type *RetTy = ToVectorTy(CI.getType(), VF);
47   SmallVector<Type *, 4> Tys;
48   for (Value *ArgOperand : CI.args())
49     Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
50   assert(!CI.getFunctionType()->isVarArg() &&
51          "VarArg functions are not supported.");
52   if (Predicate)
53     Tys.push_back(ToVectorTy(Type::getInt1Ty(RetTy->getContext()), VF));
54   FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
55   Function *VectorF =
56       Function::Create(FTy, Function::ExternalLinkage, VFName, M);
57   VectorF->copyAttributesFrom(CI.getCalledFunction());
58   ++NumVFDeclAdded;
59   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
60                     << "` of type " << *(VectorF->getType()) << "\n");
61 
62   // Make function declaration (without a body) "sticky" in the IR by
63   // listing it in the @llvm.compiler.used intrinsic.
64   assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
65                              "only on declarations.");
66   appendToCompilerUsed(*M, {VectorF});
67   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
68                     << "` to `@llvm.compiler.used`.\n");
69   ++NumCompUsedAdded;
70 }
71 
72 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
73   // This is needed to make sure we don't query the TLI for calls to
74   // bitcast of function pointers, like `%call = call i32 (i32*, ...)
75   // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
76   // as such calls make the `isFunctionVectorizable` raise an
77   // exception.
78   if (CI.isNoBuiltin() || !CI.getCalledFunction())
79     return;
80 
81   StringRef ScalarName = CI.getCalledFunction()->getName();
82 
83   // Nothing to be done if the TLI thinks the function is not
84   // vectorizable.
85   if (!TLI.isFunctionVectorizable(ScalarName))
86     return;
87   SmallVector<std::string, 8> Mappings;
88   VFABI::getVectorVariantNames(CI, Mappings);
89   Module *M = CI.getModule();
90   const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
91                                                    Mappings.end());
92 
93   auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {
94     const std::string TLIName =
95         std::string(TLI.getVectorizedFunction(ScalarName, VF, Predicate));
96     if (!TLIName.empty()) {
97       std::string MangledName = VFABI::mangleTLIVectorName(
98           TLIName, ScalarName, CI.arg_size(), VF, Predicate);
99       if (!OriginalSetOfMappings.count(MangledName)) {
100         Mappings.push_back(MangledName);
101         ++NumCallInjected;
102       }
103       Function *VariantF = M->getFunction(TLIName);
104       if (!VariantF)
105         addVariantDeclaration(CI, VF, Predicate, TLIName);
106     }
107   };
108 
109   //  All VFs in the TLI are powers of 2.
110   ElementCount WidestFixedVF, WidestScalableVF;
111   TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);
112 
113   for (bool Predicated : {false, true}) {
114     for (ElementCount VF = ElementCount::getFixed(2);
115          ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2)
116       AddVariantDecl(VF, Predicated);
117 
118     for (ElementCount VF = ElementCount::getScalable(2);
119          ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2)
120       AddVariantDecl(VF, Predicated);
121   }
122 
123   VFABI::setVectorVariantNames(&CI, Mappings);
124 }
125 
126 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
127   for (auto &I : instructions(F))
128     if (auto CI = dyn_cast<CallInst>(&I))
129       addMappingsFromTLI(TLI, *CI);
130   // Even if the pass adds IR attributes, the analyses are preserved.
131   return false;
132 }
133 
134 ////////////////////////////////////////////////////////////////////////////////
135 // New pass manager implementation.
136 ////////////////////////////////////////////////////////////////////////////////
137 PreservedAnalyses InjectTLIMappings::run(Function &F,
138                                          FunctionAnalysisManager &AM) {
139   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
140   runImpl(TLI, F);
141   // Even if the pass adds IR attributes, the analyses are preserved.
142   return PreservedAnalyses::all();
143 }
144