1 //===- NVPTXLowerAggrCopies.cpp - ------------------------------*- C++ -*--===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // \file
10 // Lower aggregate copies, memset, memcpy, memmov intrinsics into loops when
11 // the size is large or is not a compile-time constant.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "NVPTXLowerAggrCopies.h"
16 #include "llvm/Analysis/TargetTransformInfo.h"
17 #include "llvm/CodeGen/StackProtector.h"
18 #include "llvm/IR/Constants.h"
19 #include "llvm/IR/DataLayout.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/Intrinsics.h"
25 #include "llvm/IR/LLVMContext.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
29 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
30 
31 #define DEBUG_TYPE "nvptx"
32 
33 using namespace llvm;
34 
35 namespace {
36 
37 // actual analysis class, which is a functionpass
38 struct NVPTXLowerAggrCopies : public FunctionPass {
39   static char ID;
40 
41   NVPTXLowerAggrCopies() : FunctionPass(ID) {}
42 
43   void getAnalysisUsage(AnalysisUsage &AU) const override {
44     AU.addPreserved<StackProtector>();
45     AU.addRequired<TargetTransformInfoWrapperPass>();
46   }
47 
48   bool runOnFunction(Function &F) override;
49 
50   static const unsigned MaxAggrCopySize = 128;
51 
52   StringRef getPassName() const override {
53     return "Lower aggregate copies/intrinsics into loops";
54   }
55 };
56 
57 char NVPTXLowerAggrCopies::ID = 0;
58 
59 bool NVPTXLowerAggrCopies::runOnFunction(Function &F) {
60   SmallVector<LoadInst *, 4> AggrLoads;
61   SmallVector<MemIntrinsic *, 4> MemCalls;
62 
63   const DataLayout &DL = F.getParent()->getDataLayout();
64   LLVMContext &Context = F.getParent()->getContext();
65   const TargetTransformInfo &TTI =
66       getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
67 
68   // Collect all aggregate loads and mem* calls.
69   for (BasicBlock &BB : F) {
70     for (Instruction &I : BB) {
71       if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
72         if (!LI->hasOneUse())
73           continue;
74 
75         if (DL.getTypeStoreSize(LI->getType()) < MaxAggrCopySize)
76           continue;
77 
78         if (StoreInst *SI = dyn_cast<StoreInst>(LI->user_back())) {
79           if (SI->getOperand(0) != LI)
80             continue;
81           AggrLoads.push_back(LI);
82         }
83       } else if (MemIntrinsic *IntrCall = dyn_cast<MemIntrinsic>(&I)) {
84         // Convert intrinsic calls with variable size or with constant size
85         // larger than the MaxAggrCopySize threshold.
86         if (ConstantInt *LenCI = dyn_cast<ConstantInt>(IntrCall->getLength())) {
87           if (LenCI->getZExtValue() >= MaxAggrCopySize) {
88             MemCalls.push_back(IntrCall);
89           }
90         } else {
91           MemCalls.push_back(IntrCall);
92         }
93       }
94     }
95   }
96 
97   if (AggrLoads.size() == 0 && MemCalls.size() == 0) {
98     return false;
99   }
100 
101   //
102   // Do the transformation of an aggr load/copy/set to a loop
103   //
104   for (LoadInst *LI : AggrLoads) {
105     auto *SI = cast<StoreInst>(*LI->user_begin());
106     Value *SrcAddr = LI->getOperand(0);
107     Value *DstAddr = SI->getOperand(1);
108     unsigned NumLoads = DL.getTypeStoreSize(LI->getType());
109     ConstantInt *CopyLen =
110         ConstantInt::get(Type::getInt32Ty(Context), NumLoads);
111 
112     createMemCpyLoopKnownSize(/* ConvertedInst */ SI,
113                               /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr,
114                               /* CopyLen */ CopyLen,
115                               /* SrcAlign */ LI->getAlign(),
116                               /* DestAlign */ SI->getAlign(),
117                               /* SrcIsVolatile */ LI->isVolatile(),
118                               /* DstIsVolatile */ SI->isVolatile(), TTI);
119 
120     SI->eraseFromParent();
121     LI->eraseFromParent();
122   }
123 
124   // Transform mem* intrinsic calls.
125   for (MemIntrinsic *MemCall : MemCalls) {
126     if (MemCpyInst *Memcpy = dyn_cast<MemCpyInst>(MemCall)) {
127       expandMemCpyAsLoop(Memcpy, TTI);
128     } else if (MemMoveInst *Memmove = dyn_cast<MemMoveInst>(MemCall)) {
129       expandMemMoveAsLoop(Memmove);
130     } else if (MemSetInst *Memset = dyn_cast<MemSetInst>(MemCall)) {
131       expandMemSetAsLoop(Memset);
132     }
133     MemCall->eraseFromParent();
134   }
135 
136   return true;
137 }
138 
139 } // namespace
140 
141 namespace llvm {
142 void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
143 }
144 
145 INITIALIZE_PASS(NVPTXLowerAggrCopies, "nvptx-lower-aggr-copies",
146                 "Lower aggregate copies, and llvm.mem* intrinsics into loops",
147                 false, false)
148 
149 FunctionPass *llvm::createLowerAggrCopies() {
150   return new NVPTXLowerAggrCopies();
151 }
152