1480093f4SDimitry Andric //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2480093f4SDimitry Andric //
3480093f4SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4480093f4SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5480093f4SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6480093f4SDimitry Andric //
7480093f4SDimitry Andric //===----------------------------------------------------------------------===//
8480093f4SDimitry Andric //
9480093f4SDimitry Andric // Populates the VFABI attribute with the scalar-to-vector mappings
10480093f4SDimitry Andric // from the TargetLibraryInfo.
11480093f4SDimitry Andric //
12480093f4SDimitry Andric //===----------------------------------------------------------------------===//
13480093f4SDimitry Andric
14480093f4SDimitry Andric #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15480093f4SDimitry Andric #include "llvm/ADT/Statistic.h"
165ffd83dbSDimitry Andric #include "llvm/Analysis/DemandedBits.h"
175ffd83dbSDimitry Andric #include "llvm/Analysis/GlobalsModRef.h"
185ffd83dbSDimitry Andric #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19e8d8bef9SDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h"
20480093f4SDimitry Andric #include "llvm/Analysis/VectorUtils.h"
21480093f4SDimitry Andric #include "llvm/IR/InstIterator.h"
22*7a6dacacSDimitry Andric #include "llvm/IR/VFABIDemangler.h"
23480093f4SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h"
24480093f4SDimitry Andric
25480093f4SDimitry Andric using namespace llvm;
26480093f4SDimitry Andric
27480093f4SDimitry Andric #define DEBUG_TYPE "inject-tli-mappings"
28480093f4SDimitry Andric
29480093f4SDimitry Andric STATISTIC(NumCallInjected,
30480093f4SDimitry Andric "Number of calls in which the mappings have been injected.");
31480093f4SDimitry Andric
32480093f4SDimitry Andric STATISTIC(NumVFDeclAdded,
33480093f4SDimitry Andric "Number of function declarations that have been added.");
34480093f4SDimitry Andric STATISTIC(NumCompUsedAdded,
35480093f4SDimitry Andric "Number of `@llvm.compiler.used` operands that have been added.");
36480093f4SDimitry Andric
371db9f3b2SDimitry Andric /// A helper function that adds the vector variant declaration for vectorizing
381db9f3b2SDimitry Andric /// the CallInst \p CI with a vectorization factor of \p VF lanes. For each
391db9f3b2SDimitry Andric /// mapping, TLI provides a VABI prefix, which contains all information required
401db9f3b2SDimitry Andric /// to create vector function declaration.
addVariantDeclaration(CallInst & CI,const ElementCount & VF,const VecDesc * VD)41fe6060f1SDimitry Andric static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
421db9f3b2SDimitry Andric const VecDesc *VD) {
43480093f4SDimitry Andric Module *M = CI.getModule();
441db9f3b2SDimitry Andric FunctionType *ScalarFTy = CI.getFunctionType();
45480093f4SDimitry Andric
461db9f3b2SDimitry Andric assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported.");
471db9f3b2SDimitry Andric
481db9f3b2SDimitry Andric const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(
491db9f3b2SDimitry Andric VD->getVectorFunctionABIVariantString(), ScalarFTy);
501db9f3b2SDimitry Andric
511db9f3b2SDimitry Andric assert(Info && "Failed to demangle vector variant");
521db9f3b2SDimitry Andric assert(Info->Shape.VF == VF && "Mangled name does not match VF");
531db9f3b2SDimitry Andric
541db9f3b2SDimitry Andric const StringRef VFName = VD->getVectorFnName();
551db9f3b2SDimitry Andric FunctionType *VectorFTy = VFABI::createFunctionType(*Info, ScalarFTy);
561db9f3b2SDimitry Andric Function *VecFunc =
571db9f3b2SDimitry Andric Function::Create(VectorFTy, Function::ExternalLinkage, VFName, M);
581db9f3b2SDimitry Andric VecFunc->copyAttributesFrom(CI.getCalledFunction());
59480093f4SDimitry Andric ++NumVFDeclAdded;
60480093f4SDimitry Andric LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
611db9f3b2SDimitry Andric << "` of type " << *VectorFTy << "\n");
62480093f4SDimitry Andric
63480093f4SDimitry Andric // Make function declaration (without a body) "sticky" in the IR by
64480093f4SDimitry Andric // listing it in the @llvm.compiler.used intrinsic.
651db9f3b2SDimitry Andric assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` "
66480093f4SDimitry Andric "only on declarations.");
671db9f3b2SDimitry Andric appendToCompilerUsed(*M, {VecFunc});
68480093f4SDimitry Andric LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
69480093f4SDimitry Andric << "` to `@llvm.compiler.used`.\n");
70480093f4SDimitry Andric ++NumCompUsedAdded;
71480093f4SDimitry Andric }
72480093f4SDimitry Andric
addMappingsFromTLI(const TargetLibraryInfo & TLI,CallInst & CI)73480093f4SDimitry Andric static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
74480093f4SDimitry Andric // This is needed to make sure we don't query the TLI for calls to
75480093f4SDimitry Andric // bitcast of function pointers, like `%call = call i32 (i32*, ...)
76480093f4SDimitry Andric // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
77480093f4SDimitry Andric // as such calls make the `isFunctionVectorizable` raise an
78480093f4SDimitry Andric // exception.
79480093f4SDimitry Andric if (CI.isNoBuiltin() || !CI.getCalledFunction())
80480093f4SDimitry Andric return;
81480093f4SDimitry Andric
82e8d8bef9SDimitry Andric StringRef ScalarName = CI.getCalledFunction()->getName();
83e8d8bef9SDimitry Andric
84480093f4SDimitry Andric // Nothing to be done if the TLI thinks the function is not
85480093f4SDimitry Andric // vectorizable.
86480093f4SDimitry Andric if (!TLI.isFunctionVectorizable(ScalarName))
87480093f4SDimitry Andric return;
88480093f4SDimitry Andric SmallVector<std::string, 8> Mappings;
89480093f4SDimitry Andric VFABI::getVectorVariantNames(CI, Mappings);
90480093f4SDimitry Andric Module *M = CI.getModule();
91480093f4SDimitry Andric const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
92480093f4SDimitry Andric Mappings.end());
93fe6060f1SDimitry Andric
9406c3fb27SDimitry Andric auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {
955f757f3fSDimitry Andric const VecDesc *VD = TLI.getVectorMappingInfo(ScalarName, VF, Predicate);
965f757f3fSDimitry Andric if (VD && !VD->getVectorFnName().empty()) {
975f757f3fSDimitry Andric std::string MangledName = VD->getVectorFunctionABIVariantString();
98480093f4SDimitry Andric if (!OriginalSetOfMappings.count(MangledName)) {
99480093f4SDimitry Andric Mappings.push_back(MangledName);
100480093f4SDimitry Andric ++NumCallInjected;
101480093f4SDimitry Andric }
1025f757f3fSDimitry Andric Function *VariantF = M->getFunction(VD->getVectorFnName());
103480093f4SDimitry Andric if (!VariantF)
1041db9f3b2SDimitry Andric addVariantDeclaration(CI, VF, VD);
105480093f4SDimitry Andric }
106fe6060f1SDimitry Andric };
107fe6060f1SDimitry Andric
108fe6060f1SDimitry Andric // All VFs in the TLI are powers of 2.
109fe6060f1SDimitry Andric ElementCount WidestFixedVF, WidestScalableVF;
110fe6060f1SDimitry Andric TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);
111fe6060f1SDimitry Andric
11206c3fb27SDimitry Andric for (bool Predicated : {false, true}) {
113fe6060f1SDimitry Andric for (ElementCount VF = ElementCount::getFixed(2);
114fe6060f1SDimitry Andric ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2)
11506c3fb27SDimitry Andric AddVariantDecl(VF, Predicated);
116fe6060f1SDimitry Andric
11706c3fb27SDimitry Andric for (ElementCount VF = ElementCount::getScalable(2);
11806c3fb27SDimitry Andric ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2)
11906c3fb27SDimitry Andric AddVariantDecl(VF, Predicated);
12006c3fb27SDimitry Andric }
121480093f4SDimitry Andric
122480093f4SDimitry Andric VFABI::setVectorVariantNames(&CI, Mappings);
123480093f4SDimitry Andric }
124480093f4SDimitry Andric
runImpl(const TargetLibraryInfo & TLI,Function & F)125480093f4SDimitry Andric static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
126480093f4SDimitry Andric for (auto &I : instructions(F))
127480093f4SDimitry Andric if (auto CI = dyn_cast<CallInst>(&I))
128480093f4SDimitry Andric addMappingsFromTLI(TLI, *CI);
129480093f4SDimitry Andric // Even if the pass adds IR attributes, the analyses are preserved.
130480093f4SDimitry Andric return false;
131480093f4SDimitry Andric }
132480093f4SDimitry Andric
133480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
134480093f4SDimitry Andric // New pass manager implementation.
135480093f4SDimitry Andric ////////////////////////////////////////////////////////////////////////////////
run(Function & F,FunctionAnalysisManager & AM)136480093f4SDimitry Andric PreservedAnalyses InjectTLIMappings::run(Function &F,
137480093f4SDimitry Andric FunctionAnalysisManager &AM) {
138480093f4SDimitry Andric const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
139480093f4SDimitry Andric runImpl(TLI, F);
140480093f4SDimitry Andric // Even if the pass adds IR attributes, the analyses are preserved.
141480093f4SDimitry Andric return PreservedAnalyses::all();
142480093f4SDimitry Andric }
143